Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(server): load persisted vector indexes (#835)

Gautam Sharmaa96d74efb4cd
diff --git a/packages/server/src/repowise/server/app.py b/packages/server/src/repowise/server/app.py
index 94b211122..f2ce0d196 100644
--- a/packages/server/src/repowise/server/app.py
+++ b/packages/server/src/repowise/server/app.py
@@ -27,7 +27,6 @@ from repowise.core.persistence.database import (
)
from repowise.core.persistence.models import GenerationJob
from repowise.core.persistence.search import FullTextSearch
-from repowise.core.persistence.vector_store import InMemoryVectorStore
from repowise.core.providers.embedding.base import MockEmbedder
from repowise.server import __version__
from repowise.server.routers import (
@@ -190,9 +189,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
fts = FullTextSearch(engine)
await fts.ensure_index()
- # Vector store (InMemory default; LanceDB/pgvector configured via env)
+ # Reuse the repo-local LanceDB index written by CLI init/update. A fresh
+ # in-memory store is used only when this database cannot be associated with
+ # a repository or the optional LanceDB runtime is unavailable.
embedder = _build_embedder()
- vector_store = InMemoryVectorStore(embedder=embedder)
+ from repowise.server.search_helpers import build_primary_vector_store
+
+ vector_store, primary_vector_repo_id = await build_primary_vector_store(
+ session_factory,
+ db_url,
+ embedder,
+ )
# Store on app state (before scheduler, so scheduler can reference app_state)
app.state.engine = engine
@@ -200,7 +207,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.db_url = db_url
app.state.fts = fts
app.state.vector_store = vector_store
- app.state.background_tasks: set = set() # Strong refs to prevent GC of asyncio tasks
+ app.state.primary_vector_repo_id = primary_vector_repo_id
+ app.state.background_tasks = set() # Strong refs to prevent GC of asyncio tasks
app.state.job_tasks = {} # job_id → asyncio.Task (cancel endpoint)
app.state.job_cancel_tokens = {} # job_id → CancellationToken
app.state.job_events = {} # job_id → JobEventBuffer (SSE message frames)
@@ -232,6 +240,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
# repo_id → vector store (LanceDB-backed) for per-repo semantic search.
# Populated lazily by the search router on first use, then cached.
app.state.workspace_vector_stores = {} # repo_id → VectorStore
+ if primary_vector_repo_id is not None:
+ app.state.workspace_vector_stores[primary_vector_repo_id] = vector_store
try:
from pathlib import Path as _Path
diff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.py
index ce38fcd4e..ec2124182 100644
--- a/packages/server/src/repowise/server/job_executor.py
+++ b/packages/server/src/repowise/server/job_executor.py
@@ -316,7 +316,6 @@ async def execute_job(
# leaving the row stuck in 'pending' forever.
session_factory = session_factory_override or app_state.session_factory
fts = app_state.fts
- vector_store = app_state.vector_store
# ---- Fetch job + repo metadata ------------------------------------
async with get_session(session_factory) as session:
@@ -349,6 +348,21 @@ async def execute_job(
# Mark running
await update_job_status(session, job_id, "running")
+ # Vector writes and deletes must follow the repository, just like its
+ # routed SQL session. This opens/creates <repo>/.repowise/lancedb and
+ # caches it by repo id; the global primary store is only a fallback for
+ # partially initialized development app states.
+ from repowise.server.search_helpers import resolve_repo_vector_store
+
+ vector_store = await resolve_repo_vector_store(
+ app_state,
+ repo_id,
+ repo_path=repo_path,
+ create=True,
+ )
+ if vector_store is None:
+ vector_store = app_state.vector_store
+
logger.info("job_started", job_id=job_id, repo_path=repo_path, mode=mode)
# ---- Resolve LLM provider -----------------------------------------
@@ -806,9 +820,7 @@ async def _incremental_page_regen(
language=repo_cfg.get("language", "en"),
)
assembler = ContextAssembler(generation_config)
- generator = PageGenerator(
- llm_client, assembler, generation_config, repo_path=repo_path
- )
+ generator = PageGenerator(llm_client, assembler, generation_config, repo_path=repo_path)
pages = await generator.generate_all(
affected_parsed,
diff --git a/packages/server/src/repowise/server/routers/search.py b/packages/server/src/repowise/server/routers/search.py
index 0d1113594..271291b40 100644
--- a/packages/server/src/repowise/server/routers/search.py
+++ b/packages/server/src/repowise/server/routers/search.py
@@ -7,6 +7,8 @@ and vector store so a single query covers the whole workspace. Pass
from __future__ import annotations
+from pathlib import Path
+
from fastapi import APIRouter, Depends, Query, Request
from sqlalchemy import select
@@ -41,7 +43,9 @@ def _to_response(r, det_ids: set[str]) -> SearchResultResponse:
)
-async def _deterministic_page_ids(request: Request, repo_id: str | None, page_ids: list[str]) -> set[str]:
+async def _deterministic_page_ids(
+ request: Request, repo_id: str | None, page_ids: list[str]
+) -> set[str]:
"""Page ids among ``page_ids`` that are deterministic template pages.
One batch lookup against the resolved repo's DB. Best-effort: workspace
@@ -55,9 +59,7 @@ async def _deterministic_page_ids(request: Request, repo_id: str | None, page_id
factory = resolve_session_factory(request.app.state, repo_id)
async with get_session(factory) as session:
rows = await session.execute(
- select(Page.id).where(
- Page.id.in_(page_ids), Page.provider_name == "template"
- )
+ select(Page.id).where(Page.id.in_(page_ids), Page.provider_name == "template")
)
return {row[0] for row in rows.all()}
except Exception:
@@ -93,9 +95,7 @@ async def search(
if search_type == "fulltext":
results = await _fulltext(request, query, limit, repo_id=repo_id, primary_fts=fts)
else:
- results = await _semantic(
- request, query, limit, repo_id=repo_id, primary_vs=vector_store
- )
+ results = await _semantic(request, query, limit, repo_id=repo_id, primary_vs=vector_store)
det_ids = await _deterministic_page_ids(request, repo_id, [r.page_id for r in results])
return [_to_response(r, det_ids) for r in results]
@@ -148,30 +148,30 @@ async def _fulltext(request: Request, query: str, limit: int, *, repo_id, primar
async def _semantic(request: Request, query: str, limit: int, *, repo_id, primary_vs):
"""Run a semantic search across the appropriate vector store(s)."""
- from repowise.server.search_helpers import resolve_workspace_vector_store
+ from repowise.server.search_helpers import resolve_repo_vector_store
ws_config = getattr(request.app.state, "workspace_config", None)
- if ws_config is None:
- # Single-repo mode.
- return await primary_vs.search(query, limit=limit)
if repo_id is not None:
if repo_id.startswith("ws:"):
return []
- vs = await resolve_workspace_vector_store(request.app, repo_id)
+ vs = await resolve_repo_vector_store(request.app.state, repo_id)
if vs is None:
return await primary_vs.search(query, limit=limit)
return await vs.search(query, limit=limit)
+ if ws_config is None:
+ # Single-repo mode. Startup resolves the primary store from the same
+ # repo-local database used by `repowise serve`.
+ return await primary_vs.search(query, limit=limit)
+
# Fan-out: iterate over every workspace repo with an indexed wiki.db
# and try to load its persisted LanceDB store. Repos without one
# fall back to FTS so the user still gets some signal.
- from pathlib import Path as _P
-
ws_root = getattr(request.app.state, "workspace_root", None)
if ws_root is None:
return await primary_vs.search(query, limit=limit)
- ws_root_path = _P(ws_root)
+ ws_root_path = Path(ws_root)
all_results = []
workspace_fts = getattr(request.app.state, "workspace_fts", {}) or {}
@@ -186,9 +186,7 @@ async def _semantic(request: Request, query: str, limit: int, *, repo_id, primar
try:
with _sql.connect(str(db_path)) as conn:
- row = conn.execute(
- "SELECT id FROM repositories LIMIT 1"
- ).fetchone()
+ row = conn.execute("SELECT id FROM repositories LIMIT 1").fetchone()
except Exception:
row = None
rid = row[0] if row else None
@@ -198,7 +196,7 @@ async def _semantic(request: Request, query: str, limit: int, *, repo_id, primar
vs = None
if rid is not None:
try:
- vs = await resolve_workspace_vector_store(request.app, rid)
+ vs = await resolve_repo_vector_store(request.app.state, rid)
except Exception:
vs = None
if vs is not None:
diff --git a/packages/server/src/repowise/server/search_helpers.py b/packages/server/src/repowise/server/search_helpers.py
index 3fb95025f..f6bcb882e 100644
--- a/packages/server/src/repowise/server/search_helpers.py
+++ b/packages/server/src/repowise/server/search_helpers.py
@@ -1,14 +1,15 @@
-"""Helpers for search fan-out across workspace repos.
+"""Repository-aware vector-store helpers for REST search and background jobs.
These live in a separate module so they can be reused by other routers
-(chat, MCP-over-HTTP, future workspace endpoints) without each one
-re-implementing LanceDB rehydration / lazy caching.
+(chat, MCP-over-HTTP, future endpoints) without each one re-implementing
+LanceDB rehydration, routing, and lazy caching.
"""
from __future__ import annotations
import asyncio
import logging
+from contextlib import suppress
from pathlib import Path
from typing import Any
@@ -20,86 +21,150 @@ logger = logging.getLogger("repowise.server.search_helpers")
_vector_store_locks: dict[str, asyncio.Lock] = {}
-async def resolve_workspace_vector_store(app, repo_id: str) -> Any | None:
- """Return a vector store for the given workspace ``repo_id``.
+def _build_repo_vector_store(repo_path: Path, embedder: Any, *, create: bool) -> Any | None:
+ """Open a repo-local LanceDB store, with an in-memory write fallback.
+
+ Search-only callers do not create a new store when no persisted index
+ exists. Job callers pass ``create=True`` so server-generated embeddings use
+ the same durable ``.repowise/lancedb`` location as the CLI.
+ """
+ lance_dir = repo_path / ".repowise" / "lancedb"
+ if not create and not lance_dir.is_dir():
+ return None
+
+ try:
+ import lancedb # noqa: F401 # verify the optional runtime is installed
+
+ from repowise.core.persistence.vector_store import LanceDBVectorStore
+
+ if create:
+ lance_dir.mkdir(parents=True, exist_ok=True)
+ return LanceDBVectorStore(str(lance_dir), embedder=embedder)
+ except (ImportError, OSError):
+ if not create:
+ return None
+ from repowise.core.persistence.vector_store import InMemoryVectorStore
- Cached on ``app.state.workspace_vector_stores`` after first
- resolution. Returns ``None`` if:
- - We're not in workspace mode.
- - The repo_id isn't in the workspace config.
- - The repo has no ``lancedb/`` directory on disk yet.
+ logger.warning(
+ "lancedb_unavailable_using_memory",
+ extra={"repo_path": str(repo_path)},
+ )
+ return InMemoryVectorStore(embedder)
- The store uses the same embedder as the primary vector store, so a
- workspace built with the gemini embedder keeps its embeddings
- compatible across fan-out queries.
+
+async def build_primary_vector_store(
+ session_factory, db_url: str, embedder: Any
+) -> tuple[Any, str | None]:
+ """Build the primary repo's durable vector store when it can be identified.
+
+ A repo-local SQLite database normally contains one repository row. For a
+ shared registry database, only a row whose ``wiki.db`` matches ``db_url`` is
+ treated as primary. The returned repo id lets startup seed the per-repo
+ cache so jobs and searches reuse the same connection.
"""
- cache: dict | None = getattr(app.state, "workspace_vector_stores", None)
+ from sqlalchemy import select
+
+ from repowise.core.persistence.database import get_session
+ from repowise.core.persistence.models import Repository
+ from repowise.core.persistence.vector_store import InMemoryVectorStore
+
+ rows: list[tuple[str, str]] = []
+ try:
+ async with get_session(session_factory) as session:
+ result = await session.execute(select(Repository.id, Repository.local_path))
+ rows = [(row[0], row[1]) for row in result.all() if row[1]]
+ except Exception:
+ logger.debug("primary_vector_repo_lookup_failed", exc_info=True)
+
+ selected: tuple[str, str] | None = None
+ normalized_url = db_url.replace("\\", "/")
+ for repo_id, local_path in rows:
+ local_db = (Path(local_path).resolve() / ".repowise" / "wiki.db").as_posix()
+ if local_db in normalized_url:
+ selected = (repo_id, local_path)
+ break
+ if selected is None and len(rows) == 1:
+ selected = rows[0]
+
+ if selected is not None:
+ repo_id, local_path = selected
+ store = _build_repo_vector_store(Path(local_path).resolve(), embedder, create=True)
+ if store is not None:
+ logger.info(
+ "primary_vector_store_loaded",
+ extra={"repo_id": repo_id, "repo_path": local_path},
+ )
+ return store, repo_id
+
+ return InMemoryVectorStore(embedder), None
+
+
+async def resolve_repo_vector_store(
+ app_state,
+ repo_id: str,
+ *,
+ repo_path: str | Path | None = None,
+ create: bool = False,
+) -> Any | None:
+ """Return the vector store belonging to ``repo_id``.
+
+ Stores are cached by repository id. When ``repo_path`` is omitted, the
+ repository's routed session factory is used to find its canonical local
+ path, which works for workspace and API-managed repositories alike.
+ """
+ cache = getattr(app_state, "workspace_vector_stores", None)
if cache is None:
- return None
+ cache = {}
+ app_state.workspace_vector_stores = cache
if repo_id in cache:
return cache[repo_id]
lock = _vector_store_locks.setdefault(repo_id, asyncio.Lock())
async with lock:
- # Double-checked locking — another coroutine may have populated
- # the cache while we waited on the lock.
if repo_id in cache:
return cache[repo_id]
- ws_config = getattr(app.state, "workspace_config", None)
- ws_root = getattr(app.state, "workspace_root", None)
- if ws_config is None or ws_root is None:
- return None
+ if repo_path is None:
+ from repowise.core.persistence.crud import get_repository
+ from repowise.core.persistence.database import get_session
+ from repowise.server.deps import resolve_session_factory
- # Locate the repo's directory by repo_id. We don't carry an
- # alias→repo_id map directly, so scan once. Workspace sizes are
- # small (typically <20 repos), so the linear scan is fine.
- import sqlite3
-
- repo_path: Path | None = None
- ws_root_path = Path(ws_root)
- for entry in ws_config.repos:
- candidate = (ws_root_path / entry.path).resolve()
- db_path = candidate / ".repowise" / "wiki.db"
- if not db_path.exists():
- continue
try:
- with sqlite3.connect(str(db_path)) as conn:
- row = conn.execute(
- "SELECT id FROM repositories LIMIT 1"
- ).fetchone()
- if row and row[0] == repo_id:
- repo_path = candidate
- break
+ factory = resolve_session_factory(app_state, repo_id)
+ async with get_session(factory) as session:
+ repo = await get_repository(session, repo_id)
+ repo_path = repo.local_path if repo is not None else None
except Exception:
- continue
- if repo_path is None:
+ logger.debug(
+ "vector_repo_path_lookup_failed",
+ extra={"repo_id": repo_id},
+ exc_info=True,
+ )
+ return None
+
+ if not repo_path:
return None
- lance_dir = repo_path / ".repowise" / "lancedb"
- if not lance_dir.is_dir():
- return None
+ store = _build_repo_vector_store(
+ Path(repo_path).resolve(),
+ _resolve_embedder_from_state(app_state),
+ create=create,
+ )
+ if store is not None:
+ cache[repo_id] = store
+ return store
- # Build the store with the same embedder used by the primary
- # vector store. Falls back to mock when the primary is also
- # mock — keeps unit tests deterministic.
- try:
- from repowise.core.persistence.vector_store import LanceDBVectorStore
- embedder = _resolve_embedder(app)
- store = LanceDBVectorStore(str(lance_dir), embedder=embedder)
- cache[repo_id] = store
- return store
- except Exception:
- logger.debug(
- "lancedb_open_failed",
- extra={"repo_id": repo_id, "path": str(lance_dir)},
- exc_info=True,
- )
- return None
+async def resolve_workspace_vector_store(app, repo_id: str) -> Any | None:
+ """Backward-compatible app wrapper for repo vector-store resolution.
+
+ Search callers do not create a new index, so ``None`` is returned when the
+ repository has no persisted LanceDB directory yet.
+ """
+ return await resolve_repo_vector_store(app.state, repo_id)
-def _resolve_embedder(app):
+def _resolve_embedder_from_state(app_state):
"""Pull the same embedder the primary vector store was built with.
``InMemoryVectorStore`` and ``LanceDBVectorStore`` both expose
@@ -107,7 +172,7 @@ def _resolve_embedder(app):
primary store doesn't expose one — keeps semantic search "working"
against LanceDB stores built with the mock embedder during tests.
"""
- primary_vs = getattr(app.state, "vector_store", None)
+ primary_vs = getattr(app_state, "vector_store", None)
embedder = getattr(primary_vs, "_embedder", None) if primary_vs else None
if embedder is None:
from repowise.core.providers.embedding.base import MockEmbedder
@@ -121,9 +186,10 @@ async def close_workspace_vector_stores(app) -> None:
cache: dict | None = getattr(app.state, "workspace_vector_stores", None)
if not cache:
return
+ primary = getattr(app.state, "vector_store", None)
for store in list(cache.values()):
- try:
+ if store is primary:
+ continue
+ with suppress(Exception):
await store.close()
- except Exception:
- pass
cache.clear()
diff --git a/tests/unit/server/test_job_executor.py b/tests/unit/server/test_job_executor.py
index 5f422bf2d..a01f136d7 100644
--- a/tests/unit/server/test_job_executor.py
+++ b/tests/unit/server/test_job_executor.py
@@ -228,6 +228,43 @@ async def test_execute_job_merges_config_yaml_excludes(session_factory, tmp_path
assert run_pipeline_mock.await_args.kwargs["exclude_patterns"] == ["tools/"]
+async def test_execute_job_uses_repository_vector_store(session_factory, tmp_path):
+ """A workspace job must not write vectors to the global primary store."""
+ job_id = await _seed_repo_and_job(session_factory, tmp_path)
+ primary_store = object()
+ repo_store = MagicMock()
+ app_state = SimpleNamespace(
+ session_factory=session_factory,
+ fts=None,
+ vector_store=primary_store,
+ )
+ run_pipeline_mock = AsyncMock(return_value=_fake_result())
+ resolve_store = AsyncMock(return_value=repo_store)
+
+ with (
+ patch("repowise.server.job_executor.run_pipeline", run_pipeline_mock),
+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock()),
+ patch(
+ "repowise.server.search_helpers.resolve_repo_vector_store",
+ resolve_store,
+ ),
+ patch(
+ "repowise.server.provider_config.get_chat_provider_instance",
+ side_effect=RuntimeError("no provider"),
+ ),
+ ):
+ await execute_job(job_id, app_state)
+
+ resolve_store.assert_awaited_once()
+ assert resolve_store.await_args.args[0] is app_state
+ assert resolve_store.await_args.kwargs == {
+ "repo_path": str(tmp_path),
+ "create": True,
+ }
+ assert run_pipeline_mock.await_args.kwargs["vector_store"] is repo_store
+
+
@pytest.mark.asyncio
async def test_incremental_page_regen_passes_repo_path(tmp_path):
"""Incremental regen must forward repo_path to generate_all.
diff --git a/tests/unit/server/test_search_persistence.py b/tests/unit/server/test_search_persistence.py
new file mode 100644
index 000000000..ba781f48e
--- /dev/null
+++ b/tests/unit/server/test_search_persistence.py
@@ -0,0 +1,109 @@
+"""Regression tests for persisted REST semantic-search indexes."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from httpx import AsyncClient
+
+from repowise.core.persistence import crud
+from repowise.core.persistence.database import get_session
+from repowise.core.persistence.vector_store import LanceDBVectorStore
+from repowise.core.providers.embedding.base import MockEmbedder
+from repowise.server.search_helpers import (
+ build_primary_vector_store,
+ resolve_repo_vector_store,
+)
+
+
+async def test_rest_search_reopens_cli_lancedb_index(
+ client: AsyncClient,
+ app,
+ session_factory,
+ tmp_path,
+) -> None:
+ """REST search should see vectors persisted before server startup."""
+ repo_path = tmp_path / "repo"
+ lance_path = repo_path / ".repowise" / "lancedb"
+ lance_path.mkdir(parents=True)
+
+ async with get_session(session_factory) as session:
+ repo = await crud.upsert_repository(
+ session,
+ name="test-repo",
+ local_path=str(repo_path),
+ )
+ repo_id = repo.id
+
+ embedder = MockEmbedder()
+ cli_store = LanceDBVectorStore(str(lance_path), embedder=embedder)
+ await cli_store.embed_and_upsert(
+ "file_page:src/auth.py",
+ "Authentication module handles user login and logout",
+ {
+ "title": "auth.py",
+ "page_type": "file_page",
+ "target_path": "src/auth.py",
+ "content": "Authentication module handles user login and logout",
+ },
+ )
+ await cli_store.close()
+
+ loaded_store, loaded_repo_id = await build_primary_vector_store(
+ session_factory,
+ "sqlite+aiosqlite:///:memory:",
+ embedder,
+ )
+ original_store = app.state.vector_store
+ original_cache = getattr(app.state, "workspace_vector_stores", None)
+ app.state.vector_store = loaded_store
+ app.state.workspace_vector_stores = {repo_id: loaded_store}
+ try:
+ response = await client.get(
+ "/api/search",
+ params={"query": "authentication", "search_type": "semantic"},
+ )
+ finally:
+ await loaded_store.close()
+ app.state.vector_store = original_store
+ app.state.workspace_vector_stores = original_cache
+
+ assert loaded_repo_id == repo_id
+ assert response.status_code == 200
+ assert response.json()[0]["page_id"] == "file_page:src/auth.py"
+
+
+async def test_repo_vector_stores_are_isolated(tmp_path) -> None:
+ """Workspace jobs should persist vectors in their own repo directories."""
+ embedder = MockEmbedder()
+ state = SimpleNamespace(
+ vector_store=SimpleNamespace(_embedder=embedder),
+ workspace_vector_stores={},
+ )
+ repo_a = tmp_path / "repo-a"
+ repo_b = tmp_path / "repo-b"
+
+ store_a = await resolve_repo_vector_store(
+ state,
+ "repo-a",
+ repo_path=repo_a,
+ create=True,
+ )
+ store_b = await resolve_repo_vector_store(
+ state,
+ "repo-b",
+ repo_path=repo_b,
+ create=True,
+ )
+
+ try:
+ await store_a.embed_and_upsert("page-a", "alpha", {"title": "A"})
+ await store_b.embed_and_upsert("page-b", "beta", {"title": "B"})
+
+ assert await store_a.list_page_ids() == {"page-a"}
+ assert await store_b.list_page_ids() == {"page-b"}
+ assert (repo_a / ".repowise" / "lancedb").is_dir()
+ assert (repo_b / ".repowise" / "lancedb").is_dir()
+ finally:
+ await store_a.close()
+ await store_b.close()