Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(server): load persisted vector indexes (#835)
Gautam Sharmaa96d74efb4cd
diff --git a/packages/server/src/repowise/server/app.py b/packages/server/src/repowise/server/app.pyindex 94b211122..f2ce0d196 100644--- a/packages/server/src/repowise/server/app.py+++ b/packages/server/src/repowise/server/app.py@@ -27,7 +27,6 @@ from repowise.core.persistence.database import ()from repowise.core.persistence.models import GenerationJobfrom repowise.core.persistence.search import FullTextSearch-from repowise.core.persistence.vector_store import InMemoryVectorStorefrom repowise.core.providers.embedding.base import MockEmbedderfrom repowise.server import __version__from repowise.server.routers import (@@ -190,9 +189,17 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:fts = FullTextSearch(engine)await fts.ensure_index()- # Vector store (InMemory default; LanceDB/pgvector configured via env)+ # Reuse the repo-local LanceDB index written by CLI init/update. A fresh+ # in-memory store is used only when this database cannot be associated with+ # a repository or the optional LanceDB runtime is unavailable.embedder = _build_embedder()- vector_store = InMemoryVectorStore(embedder=embedder)+ from repowise.server.search_helpers import build_primary_vector_store++ vector_store, primary_vector_repo_id = await build_primary_vector_store(+ session_factory,+ db_url,+ embedder,+ )# Store on app state (before scheduler, so scheduler can reference app_state)app.state.engine = engine@@ -200,7 +207,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:app.state.db_url = db_urlapp.state.fts = ftsapp.state.vector_store = vector_store- app.state.background_tasks: set = set() # Strong refs to prevent GC of asyncio tasks+ app.state.primary_vector_repo_id = primary_vector_repo_id+ app.state.background_tasks = set() # Strong refs to prevent GC of asyncio tasksapp.state.job_tasks = {} # job_id → asyncio.Task (cancel endpoint)app.state.job_cancel_tokens = {} # job_id → CancellationTokenapp.state.job_events = {} # job_id → JobEventBuffer (SSE message frames)@@ -232,6 +240,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:# repo_id → vector store (LanceDB-backed) for per-repo semantic search.# Populated lazily by the search router on first use, then cached.app.state.workspace_vector_stores = {} # repo_id → VectorStore+ if primary_vector_repo_id is not None:+ app.state.workspace_vector_stores[primary_vector_repo_id] = vector_storetry:from pathlib import Path as _Pathdiff --git a/packages/server/src/repowise/server/job_executor.py b/packages/server/src/repowise/server/job_executor.pyindex ce38fcd4e..ec2124182 100644--- a/packages/server/src/repowise/server/job_executor.py+++ b/packages/server/src/repowise/server/job_executor.py@@ -316,7 +316,6 @@ async def execute_job(# leaving the row stuck in 'pending' forever.session_factory = session_factory_override or app_state.session_factoryfts = app_state.fts- vector_store = app_state.vector_store# ---- Fetch job + repo metadata ------------------------------------async with get_session(session_factory) as session:@@ -349,6 +348,21 @@ async def execute_job(# Mark runningawait update_job_status(session, job_id, "running")+ # Vector writes and deletes must follow the repository, just like its+ # routed SQL session. This opens/creates <repo>/.repowise/lancedb and+ # caches it by repo id; the global primary store is only a fallback for+ # partially initialized development app states.+ from repowise.server.search_helpers import resolve_repo_vector_store++ vector_store = await resolve_repo_vector_store(+ app_state,+ repo_id,+ repo_path=repo_path,+ create=True,+ )+ if vector_store is None:+ vector_store = app_state.vector_store+logger.info("job_started", job_id=job_id, repo_path=repo_path, mode=mode)# ---- Resolve LLM provider -----------------------------------------@@ -806,9 +820,7 @@ async def _incremental_page_regen(language=repo_cfg.get("language", "en"),)assembler = ContextAssembler(generation_config)- generator = PageGenerator(- llm_client, assembler, generation_config, repo_path=repo_path- )+ generator = PageGenerator(llm_client, assembler, generation_config, repo_path=repo_path)pages = await generator.generate_all(affected_parsed,diff --git a/packages/server/src/repowise/server/routers/search.py b/packages/server/src/repowise/server/routers/search.pyindex 0d1113594..271291b40 100644--- a/packages/server/src/repowise/server/routers/search.py+++ b/packages/server/src/repowise/server/routers/search.py@@ -7,6 +7,8 @@ and vector store so a single query covers the whole workspace. Passfrom __future__ import annotations+from pathlib import Path+from fastapi import APIRouter, Depends, Query, Requestfrom sqlalchemy import select@@ -41,7 +43,9 @@ def _to_response(r, det_ids: set[str]) -> SearchResultResponse:)-async def _deterministic_page_ids(request: Request, repo_id: str | None, page_ids: list[str]) -> set[str]:+async def _deterministic_page_ids(+ request: Request, repo_id: str | None, page_ids: list[str]+) -> set[str]:"""Page ids among ``page_ids`` that are deterministic template pages.One batch lookup against the resolved repo's DB. Best-effort: workspace@@ -55,9 +59,7 @@ async def _deterministic_page_ids(request: Request, repo_id: str | None, page_idfactory = resolve_session_factory(request.app.state, repo_id)async with get_session(factory) as session:rows = await session.execute(- select(Page.id).where(- Page.id.in_(page_ids), Page.provider_name == "template"- )+ select(Page.id).where(Page.id.in_(page_ids), Page.provider_name == "template"))return {row[0] for row in rows.all()}except Exception:@@ -93,9 +95,7 @@ async def search(if search_type == "fulltext":results = await _fulltext(request, query, limit, repo_id=repo_id, primary_fts=fts)else:- results = await _semantic(- request, query, limit, repo_id=repo_id, primary_vs=vector_store- )+ results = await _semantic(request, query, limit, repo_id=repo_id, primary_vs=vector_store)det_ids = await _deterministic_page_ids(request, repo_id, [r.page_id for r in results])return [_to_response(r, det_ids) for r in results]@@ -148,30 +148,30 @@ async def _fulltext(request: Request, query: str, limit: int, *, repo_id, primarasync def _semantic(request: Request, query: str, limit: int, *, repo_id, primary_vs):"""Run a semantic search across the appropriate vector store(s)."""- from repowise.server.search_helpers import resolve_workspace_vector_store+ from repowise.server.search_helpers import resolve_repo_vector_storews_config = getattr(request.app.state, "workspace_config", None)- if ws_config is None:- # Single-repo mode.- return await primary_vs.search(query, limit=limit)if repo_id is not None:if repo_id.startswith("ws:"):return []- vs = await resolve_workspace_vector_store(request.app, repo_id)+ vs = await resolve_repo_vector_store(request.app.state, repo_id)if vs is None:return await primary_vs.search(query, limit=limit)return await vs.search(query, limit=limit)+ if ws_config is None:+ # Single-repo mode. Startup resolves the primary store from the same+ # repo-local database used by `repowise serve`.+ return await primary_vs.search(query, limit=limit)+# Fan-out: iterate over every workspace repo with an indexed wiki.db# and try to load its persisted LanceDB store. Repos without one# fall back to FTS so the user still gets some signal.- from pathlib import Path as _P-ws_root = getattr(request.app.state, "workspace_root", None)if ws_root is None:return await primary_vs.search(query, limit=limit)- ws_root_path = _P(ws_root)+ ws_root_path = Path(ws_root)all_results = []workspace_fts = getattr(request.app.state, "workspace_fts", {}) or {}@@ -186,9 +186,7 @@ async def _semantic(request: Request, query: str, limit: int, *, repo_id, primartry:with _sql.connect(str(db_path)) as conn:- row = conn.execute(- "SELECT id FROM repositories LIMIT 1"- ).fetchone()+ row = conn.execute("SELECT id FROM repositories LIMIT 1").fetchone()except Exception:row = Nonerid = row[0] if row else None@@ -198,7 +196,7 @@ async def _semantic(request: Request, query: str, limit: int, *, repo_id, primarvs = Noneif rid is not None:try:- vs = await resolve_workspace_vector_store(request.app, rid)+ vs = await resolve_repo_vector_store(request.app.state, rid)except Exception:vs = Noneif vs is not None:diff --git a/packages/server/src/repowise/server/search_helpers.py b/packages/server/src/repowise/server/search_helpers.pyindex 3fb95025f..f6bcb882e 100644--- a/packages/server/src/repowise/server/search_helpers.py+++ b/packages/server/src/repowise/server/search_helpers.py@@ -1,14 +1,15 @@-"""Helpers for search fan-out across workspace repos.+"""Repository-aware vector-store helpers for REST search and background jobs.These live in a separate module so they can be reused by other routers-(chat, MCP-over-HTTP, future workspace endpoints) without each one-re-implementing LanceDB rehydration / lazy caching.+(chat, MCP-over-HTTP, future endpoints) without each one re-implementing+LanceDB rehydration, routing, and lazy caching."""from __future__ import annotationsimport asyncioimport logging+from contextlib import suppressfrom pathlib import Pathfrom typing import Any@@ -20,86 +21,150 @@ logger = logging.getLogger("repowise.server.search_helpers")_vector_store_locks: dict[str, asyncio.Lock] = {}-async def resolve_workspace_vector_store(app, repo_id: str) -> Any | None:- """Return a vector store for the given workspace ``repo_id``.+def _build_repo_vector_store(repo_path: Path, embedder: Any, *, create: bool) -> Any | None:+ """Open a repo-local LanceDB store, with an in-memory write fallback.++ Search-only callers do not create a new store when no persisted index+ exists. Job callers pass ``create=True`` so server-generated embeddings use+ the same durable ``.repowise/lancedb`` location as the CLI.+ """+ lance_dir = repo_path / ".repowise" / "lancedb"+ if not create and not lance_dir.is_dir():+ return None++ try:+ import lancedb # noqa: F401 # verify the optional runtime is installed++ from repowise.core.persistence.vector_store import LanceDBVectorStore++ if create:+ lance_dir.mkdir(parents=True, exist_ok=True)+ return LanceDBVectorStore(str(lance_dir), embedder=embedder)+ except (ImportError, OSError):+ if not create:+ return None+ from repowise.core.persistence.vector_store import InMemoryVectorStore- Cached on ``app.state.workspace_vector_stores`` after first- resolution. Returns ``None`` if:- - We're not in workspace mode.- - The repo_id isn't in the workspace config.- - The repo has no ``lancedb/`` directory on disk yet.+ logger.warning(+ "lancedb_unavailable_using_memory",+ extra={"repo_path": str(repo_path)},+ )+ return InMemoryVectorStore(embedder)- The store uses the same embedder as the primary vector store, so a- workspace built with the gemini embedder keeps its embeddings- compatible across fan-out queries.++async def build_primary_vector_store(+ session_factory, db_url: str, embedder: Any+) -> tuple[Any, str | None]:+ """Build the primary repo's durable vector store when it can be identified.++ A repo-local SQLite database normally contains one repository row. For a+ shared registry database, only a row whose ``wiki.db`` matches ``db_url`` is+ treated as primary. The returned repo id lets startup seed the per-repo+ cache so jobs and searches reuse the same connection."""- cache: dict | None = getattr(app.state, "workspace_vector_stores", None)+ from sqlalchemy import select++ from repowise.core.persistence.database import get_session+ from repowise.core.persistence.models import Repository+ from repowise.core.persistence.vector_store import InMemoryVectorStore++ rows: list[tuple[str, str]] = []+ try:+ async with get_session(session_factory) as session:+ result = await session.execute(select(Repository.id, Repository.local_path))+ rows = [(row[0], row[1]) for row in result.all() if row[1]]+ except Exception:+ logger.debug("primary_vector_repo_lookup_failed", exc_info=True)++ selected: tuple[str, str] | None = None+ normalized_url = db_url.replace("\\", "/")+ for repo_id, local_path in rows:+ local_db = (Path(local_path).resolve() / ".repowise" / "wiki.db").as_posix()+ if local_db in normalized_url:+ selected = (repo_id, local_path)+ break+ if selected is None and len(rows) == 1:+ selected = rows[0]++ if selected is not None:+ repo_id, local_path = selected+ store = _build_repo_vector_store(Path(local_path).resolve(), embedder, create=True)+ if store is not None:+ logger.info(+ "primary_vector_store_loaded",+ extra={"repo_id": repo_id, "repo_path": local_path},+ )+ return store, repo_id++ return InMemoryVectorStore(embedder), None+++async def resolve_repo_vector_store(+ app_state,+ repo_id: str,+ *,+ repo_path: str | Path | None = None,+ create: bool = False,+) -> Any | None:+ """Return the vector store belonging to ``repo_id``.++ Stores are cached by repository id. When ``repo_path`` is omitted, the+ repository's routed session factory is used to find its canonical local+ path, which works for workspace and API-managed repositories alike.+ """+ cache = getattr(app_state, "workspace_vector_stores", None)if cache is None:- return None+ cache = {}+ app_state.workspace_vector_stores = cacheif repo_id in cache:return cache[repo_id]lock = _vector_store_locks.setdefault(repo_id, asyncio.Lock())async with lock:- # Double-checked locking — another coroutine may have populated- # the cache while we waited on the lock.if repo_id in cache:return cache[repo_id]- ws_config = getattr(app.state, "workspace_config", None)- ws_root = getattr(app.state, "workspace_root", None)- if ws_config is None or ws_root is None:- return None+ if repo_path is None:+ from repowise.core.persistence.crud import get_repository+ from repowise.core.persistence.database import get_session+ from repowise.server.deps import resolve_session_factory- # Locate the repo's directory by repo_id. We don't carry an- # alias→repo_id map directly, so scan once. Workspace sizes are- # small (typically <20 repos), so the linear scan is fine.- import sqlite3-- repo_path: Path | None = None- ws_root_path = Path(ws_root)- for entry in ws_config.repos:- candidate = (ws_root_path / entry.path).resolve()- db_path = candidate / ".repowise" / "wiki.db"- if not db_path.exists():- continuetry:- with sqlite3.connect(str(db_path)) as conn:- row = conn.execute(- "SELECT id FROM repositories LIMIT 1"- ).fetchone()- if row and row[0] == repo_id:- repo_path = candidate- break+ factory = resolve_session_factory(app_state, repo_id)+ async with get_session(factory) as session:+ repo = await get_repository(session, repo_id)+ repo_path = repo.local_path if repo is not None else Noneexcept Exception:- continue- if repo_path is None:+ logger.debug(+ "vector_repo_path_lookup_failed",+ extra={"repo_id": repo_id},+ exc_info=True,+ )+ return None++ if not repo_path:return None- lance_dir = repo_path / ".repowise" / "lancedb"- if not lance_dir.is_dir():- return None+ store = _build_repo_vector_store(+ Path(repo_path).resolve(),+ _resolve_embedder_from_state(app_state),+ create=create,+ )+ if store is not None:+ cache[repo_id] = store+ return store- # Build the store with the same embedder used by the primary- # vector store. Falls back to mock when the primary is also- # mock — keeps unit tests deterministic.- try:- from repowise.core.persistence.vector_store import LanceDBVectorStore- embedder = _resolve_embedder(app)- store = LanceDBVectorStore(str(lance_dir), embedder=embedder)- cache[repo_id] = store- return store- except Exception:- logger.debug(- "lancedb_open_failed",- extra={"repo_id": repo_id, "path": str(lance_dir)},- exc_info=True,- )- return None+async def resolve_workspace_vector_store(app, repo_id: str) -> Any | None:+ """Backward-compatible app wrapper for repo vector-store resolution.++ Search callers do not create a new index, so ``None`` is returned when the+ repository has no persisted LanceDB directory yet.+ """+ return await resolve_repo_vector_store(app.state, repo_id)-def _resolve_embedder(app):+def _resolve_embedder_from_state(app_state):"""Pull the same embedder the primary vector store was built with.``InMemoryVectorStore`` and ``LanceDBVectorStore`` both expose@@ -107,7 +172,7 @@ def _resolve_embedder(app):primary store doesn't expose one — keeps semantic search "working"against LanceDB stores built with the mock embedder during tests."""- primary_vs = getattr(app.state, "vector_store", None)+ primary_vs = getattr(app_state, "vector_store", None)embedder = getattr(primary_vs, "_embedder", None) if primary_vs else Noneif embedder is None:from repowise.core.providers.embedding.base import MockEmbedder@@ -121,9 +186,10 @@ async def close_workspace_vector_stores(app) -> None:cache: dict | None = getattr(app.state, "workspace_vector_stores", None)if not cache:return+ primary = getattr(app.state, "vector_store", None)for store in list(cache.values()):- try:+ if store is primary:+ continue+ with suppress(Exception):await store.close()- except Exception:- passcache.clear()diff --git a/tests/unit/server/test_job_executor.py b/tests/unit/server/test_job_executor.pyindex 5f422bf2d..a01f136d7 100644--- a/tests/unit/server/test_job_executor.py+++ b/tests/unit/server/test_job_executor.py@@ -228,6 +228,43 @@ async def test_execute_job_merges_config_yaml_excludes(session_factory, tmp_pathassert run_pipeline_mock.await_args.kwargs["exclude_patterns"] == ["tools/"]+async def test_execute_job_uses_repository_vector_store(session_factory, tmp_path):+ """A workspace job must not write vectors to the global primary store."""+ job_id = await _seed_repo_and_job(session_factory, tmp_path)+ primary_store = object()+ repo_store = MagicMock()+ app_state = SimpleNamespace(+ session_factory=session_factory,+ fts=None,+ vector_store=primary_store,+ )+ run_pipeline_mock = AsyncMock(return_value=_fake_result())+ resolve_store = AsyncMock(return_value=repo_store)++ with (+ patch("repowise.server.job_executor.run_pipeline", run_pipeline_mock),+ patch("repowise.server.job_executor.persist_pipeline_result", AsyncMock()),+ patch(+ "repowise.server.search_helpers.resolve_repo_vector_store",+ resolve_store,+ ),+ patch(+ "repowise.server.provider_config.get_chat_provider_instance",+ side_effect=RuntimeError("no provider"),+ ),+ ):+ await execute_job(job_id, app_state)++ resolve_store.assert_awaited_once()+ assert resolve_store.await_args.args[0] is app_state+ assert resolve_store.await_args.kwargs == {+ "repo_path": str(tmp_path),+ "create": True,+ }+ assert run_pipeline_mock.await_args.kwargs["vector_store"] is repo_store++@pytest.mark.asyncioasync def test_incremental_page_regen_passes_repo_path(tmp_path):"""Incremental regen must forward repo_path to generate_all.diff --git a/tests/unit/server/test_search_persistence.py b/tests/unit/server/test_search_persistence.pynew file mode 100644index 000000000..ba781f48e--- /dev/null+++ b/tests/unit/server/test_search_persistence.py@@ -0,0 +1,109 @@+"""Regression tests for persisted REST semantic-search indexes."""++from __future__ import annotations++from types import SimpleNamespace++from httpx import AsyncClient++from repowise.core.persistence import crud+from repowise.core.persistence.database import get_session+from repowise.core.persistence.vector_store import LanceDBVectorStore+from repowise.core.providers.embedding.base import MockEmbedder+from repowise.server.search_helpers import (+ build_primary_vector_store,+ resolve_repo_vector_store,+)+++async def test_rest_search_reopens_cli_lancedb_index(+ client: AsyncClient,+ app,+ session_factory,+ tmp_path,+) -> None:+ """REST search should see vectors persisted before server startup."""+ repo_path = tmp_path / "repo"+ lance_path = repo_path / ".repowise" / "lancedb"+ lance_path.mkdir(parents=True)++ async with get_session(session_factory) as session:+ repo = await crud.upsert_repository(+ session,+ name="test-repo",+ local_path=str(repo_path),+ )+ repo_id = repo.id++ embedder = MockEmbedder()+ cli_store = LanceDBVectorStore(str(lance_path), embedder=embedder)+ await cli_store.embed_and_upsert(+ "file_page:src/auth.py",+ "Authentication module handles user login and logout",+ {+ "title": "auth.py",+ "page_type": "file_page",+ "target_path": "src/auth.py",+ "content": "Authentication module handles user login and logout",+ },+ )+ await cli_store.close()++ loaded_store, loaded_repo_id = await build_primary_vector_store(+ session_factory,+ "sqlite+aiosqlite:///:memory:",+ embedder,+ )+ original_store = app.state.vector_store+ original_cache = getattr(app.state, "workspace_vector_stores", None)+ app.state.vector_store = loaded_store+ app.state.workspace_vector_stores = {repo_id: loaded_store}+ try:+ response = await client.get(+ "/api/search",+ params={"query": "authentication", "search_type": "semantic"},+ )+ finally:+ await loaded_store.close()+ app.state.vector_store = original_store+ app.state.workspace_vector_stores = original_cache++ assert loaded_repo_id == repo_id+ assert response.status_code == 200+ assert response.json()[0]["page_id"] == "file_page:src/auth.py"+++async def test_repo_vector_stores_are_isolated(tmp_path) -> None:+ """Workspace jobs should persist vectors in their own repo directories."""+ embedder = MockEmbedder()+ state = SimpleNamespace(+ vector_store=SimpleNamespace(_embedder=embedder),+ workspace_vector_stores={},+ )+ repo_a = tmp_path / "repo-a"+ repo_b = tmp_path / "repo-b"++ store_a = await resolve_repo_vector_store(+ state,+ "repo-a",+ repo_path=repo_a,+ create=True,+ )+ store_b = await resolve_repo_vector_store(+ state,+ "repo-b",+ repo_path=repo_b,+ create=True,+ )++ try:+ await store_a.embed_and_upsert("page-a", "alpha", {"title": "A"})+ await store_b.embed_and_upsert("page-b", "beta", {"title": "B"})++ assert await store_a.list_page_ids() == {"page-a"}+ assert await store_b.list_page_ids() == {"page-b"}+ assert (repo_a / ".repowise" / "lancedb").is_dir()+ assert (repo_b / ".repowise" / "lancedb").is_dir()+ finally:+ await store_a.close()+ await store_b.close()