Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
perf(mcp): cache query embeddings so a repeated search skips the provider (#2189)
Raghav Chamadiya5ffc478120ca
diff --git a/packages/core/src/repowise/core/providers/embedding/caching.py b/packages/core/src/repowise/core/providers/embedding/caching.pynew file mode 100644index 000000000..6220559a6--- /dev/null+++ b/packages/core/src/repowise/core/providers/embedding/caching.py@@ -0,0 +1,66 @@+"""Memoize query embeddings so a repeated search does not pay the provider twice.++Every ``search_codebase`` call embeds its query text through the configured+provider — a network round trip, measured at ~215ms against OpenAI, on a path+where the vector lookup itself costs ~35ms. Agents repeat queries within a+session, and each repeat paid that round trip again.++Wrap the embedder rather than caching inside one provider or one store: the+query paths reach the embedder through three different store methods+(``search``, ``search_many``, ``embed_texts``), and there are five providers+with the same shape, so this is the one layer common to all of them.+"""++from __future__ import annotations++import hashlib+from collections import OrderedDict+from typing import Any++#: Entries kept per wrapped embedder. A 1536-dimension vector costs roughly+#: 40KB live as a Python list, so this bounds one embedder at a few megabytes.+MAX_ENTRIES = 256+++class CachingEmbedder:+ """An :class:`~repowise.core.providers.embedding.base.Embedder` that memoizes single texts.++ A cached vector is a pure function of (embedder configuration, text), so a+ hit is exactly as correct as a fresh call and never goes stale. The cache+ lives on the instance, so the configuration behind it is fixed for the life+ of every entry and the text alone identifies a vector.++ The gate is batch size, not caller: only single-text calls are cached. That+ is a proxy rather than a rule — a single-document write is cached too, and+ harmlessly, since the vector is the same either way. What it reliably keeps+ out is the bulk path, where hundreds of corpus strings that are never+ queried again would evict the handful of real queries this exists for.+ """++ def __init__(self, inner: Any) -> None:+ self._inner = inner+ self._cache: OrderedDict[str, list[float]] = OrderedDict()++ @property+ def dimensions(self) -> int:+ return self._inner.dimensions++ async def embed(self, texts: list[str]) -> list[list[float]]:+ if len(texts) != 1:+ return await self._inner.embed(texts)++ # Hashed rather than stored: a query reaches 30_000 characters upstream,+ # and holding the raw text would cost more than the vector it caches.+ key = hashlib.sha256(texts[0].encode()).hexdigest()+ cached = self._cache.get(key)+ if cached is not None:+ self._cache.move_to_end(key)+ # Copy: a caller that mutates its vector must not edit the entry.+ return [list(cached)]++ vectors = await self._inner.embed(texts)+ if len(vectors) == 1:+ self._cache[key] = list(vectors[0])+ while len(self._cache) > MAX_ENTRIES:+ self._cache.popitem(last=False)+ return vectorsdiff --git a/packages/server/src/repowise/server/mcp_server/_server.py b/packages/server/src/repowise/server/mcp_server/_server.pyindex 10a584512..de5acebd0 100644--- a/packages/server/src/repowise/server/mcp_server/_server.py+++ b/packages/server/src/repowise/server/mcp_server/_server.py@@ -24,7 +24,8 @@ from repowise.core.persistence.database import (from repowise.core.persistence.search import FullTextSearchfrom repowise.core.persistence.vector_store import InMemoryVectorStorefrom repowise.core.platform.telemetry import GROUP_LEAF_TYPES_ATTR-from repowise.core.providers.embedding.base import KeylessEmbedder+from repowise.core.providers.embedding.base import KeylessEmbedder, is_semantic_embedder+from repowise.core.providers.embedding.caching import CachingEmbedderfrom repowise.server.mcp_server import _state_log = __import__("logging").getLogger("repowise.mcp")@@ -289,6 +290,23 @@ def _resolve_embedder():return KeylessEmbedder()+def _query_embedder():+ """The embedder this server answers queries with.++ Wrapped so a repeated query does not pay the provider round trip again.++ Only this process wraps: a CLI invocation is one-shot and serves a single+ query, so a per-process cache could never hit there. The HTTP server+ (``server/app.py``) is long-lived and would benefit, and is left unwrapped+ only to keep this change to the surface the cost was measured on.++ Keyless is left bare — :func:`is_semantic_embedder` identifies it by type+ to switch the vector leg off, and a wrapper would defeat that.+ """+ embedder = _resolve_embedder()+ return CachingEmbedder(embedder) if is_semantic_embedder(embedder) else embedder++#: How often the running server re-reads release currency. The PyPI fetch#: itself is TTL-cached on disk for a day and shared with the CLI advisory, so#: this bounds only the in-process refresh, never the network.@@ -383,7 +401,7 @@ async def _load_vector_stores(repo_path: str | None) -> None:import asyncio as _asynciotry:- embedder = _resolve_embedder()+ embedder = _query_embedder()vector_store: Any = InMemoryVectorStore(embedder=embedder)try:@@ -510,7 +528,7 @@ async def _lifespan(server: FastMCP):registry = RepoRegistry(workspace_root=ws_root,ws_config=ws_config,- embedder_factory=lambda: _resolve_embedder(),+ embedder_factory=_query_embedder,)# Eagerly load the default repo so tools work immediately. A failurediff --git a/tests/unit/server/mcp/test_embedder_resolution.py b/tests/unit/server/mcp/test_embedder_resolution.pyindex 5173a825a..6043d2368 100644--- a/tests/unit/server/mcp/test_embedder_resolution.py+++ b/tests/unit/server/mcp/test_embedder_resolution.py@@ -16,7 +16,12 @@ from pathlib import Pathimport pytest-from repowise.core.providers.embedding.base import MockEmbedder+from repowise.core.providers.embedding.base import (+ KeylessEmbedder,+ MockEmbedder,+ is_semantic_embedder,+)+from repowise.core.providers.embedding.caching import CachingEmbedderfrom repowise.server.mcp_server import _server, _statefrom repowise.server.mcp_server._meta import build_meta@@ -330,3 +335,39 @@ def test_build_meta_omits_degraded_when_embedder_unresolved(monkeypatch):meta = build_meta(timing_ms=1.0)assert "embedder_degraded" not in metaassert "semantic_search" not in meta+++class TestQueryEmbedderWrapping:+ """``_query_embedder`` is what the server's stores actually embed with.++ It memoizes repeated queries, but must never wrap a keyless embedder:+ ``is_semantic_embedder`` identifies that one by type in order to switch the+ vector leg off, and a wrapper would report as semantic and silently+ reactivate a leg whose vectors carry no signal.+ """++ def test_configured_embedder_is_wrapped_for_caching(self, monkeypatch, tmp_path):+ monkeypatch.setenv("REPOWISE_EMBEDDER", "openai")+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")++ embedder = _server._query_embedder()++ assert isinstance(embedder, CachingEmbedder)+ assert is_semantic_embedder(embedder)++ def test_unconfigured_keyless_is_left_unwrapped(self):+ embedder = _server._query_embedder()++ assert isinstance(embedder, KeylessEmbedder)+ assert not is_semantic_embedder(embedder), "the vector leg must stay off"++ def test_degraded_fallback_is_left_unwrapped(self, monkeypatch):+ """A configured embedder that cannot initialise falls back to keyless —+ wrapping that would hide it from the vector-leg check."""+ monkeypatch.setenv("REPOWISE_EMBEDDER", "openai")++ embedder = _server._query_embedder()++ assert _state._embedder_status["degraded"] is True+ assert isinstance(embedder, KeylessEmbedder)+ assert not is_semantic_embedder(embedder)diff --git a/tests/unit/test_persistence/test_caching_embedder.py b/tests/unit/test_persistence/test_caching_embedder.pynew file mode 100644index 000000000..e1ca5e2a6--- /dev/null+++ b/tests/unit/test_persistence/test_caching_embedder.py@@ -0,0 +1,138 @@+"""Tests for the query-embedding cache wrapper.++Every ``search_codebase`` embeds its query through the provider — a network+round trip on a path where the vector lookup itself is an order of magnitude+cheaper. These pin that a repeat is served locally, and that the cache cannot+hand back a vector from a different embedder or a different width.+"""++from __future__ import annotations++from repowise.core.providers.embedding.caching import MAX_ENTRIES, CachingEmbedder+++class _CountingEmbedder:+ """Records every batch it is asked to embed."""++ def __init__(self, dimensions: int = 4, seed: float = 1.0) -> None:+ self._dimensions = dimensions+ self._seed = seed+ self.calls: list[list[str]] = []++ @property+ def dimensions(self) -> int:+ return self._dimensions++ async def embed(self, texts: list[str]) -> list[list[float]]:+ self.calls.append(list(texts))+ return [[self._seed * (i + 1)] * self._dimensions for i, _ in enumerate(texts)]+++async def test_repeated_query_is_served_from_cache():+ inner = _CountingEmbedder()+ embedder = CachingEmbedder(inner)++ first = await embedder.embed(["auth service"])+ second = await embedder.embed(["auth service"])++ assert first == second+ assert inner.calls == [["auth service"]], "the provider was asked twice"+++async def test_distinct_queries_each_reach_the_provider():+ inner = _CountingEmbedder()+ embedder = CachingEmbedder(inner)++ await embedder.embed(["one"])+ await embedder.embed(["two"])++ assert inner.calls == [["one"], ["two"]]+++async def test_multi_text_batches_are_not_cached():+ """Bulk calls are indexing or decision matching, never repeated queries."""+ inner = _CountingEmbedder()+ embedder = CachingEmbedder(inner)++ await embedder.embed(["a", "b"])+ await embedder.embed(["a", "b"])++ assert inner.calls == [["a", "b"], ["a", "b"]]+++async def test_a_bulk_call_does_not_evict_or_serve_queries():+ inner = _CountingEmbedder()+ embedder = CachingEmbedder(inner)++ await embedder.embed(["query"])+ await embedder.embed(["query", "other"])+ await embedder.embed(["query"])++ # The single-text query was embedded once; the batch neither used nor+ # displaced its entry.+ assert inner.calls == [["query"], ["query", "other"]]+++async def test_cache_is_bounded_and_evicts_least_recently_used():+ """A per-query entry that is never evicted is a leak in a long-lived server."""+ inner = _CountingEmbedder()+ embedder = CachingEmbedder(inner)++ await embedder.embed(["oldest"])+ for i in range(MAX_ENTRIES - 1):+ await embedder.embed([f"filler-{i}"])+ await embedder.embed(["oldest"]) # refresh, so a filler is now least recent+ await embedder.embed(["overflow"]) # pushes past the cap++ assert len(embedder._cache) == MAX_ENTRIES+ inner.calls.clear()+ await embedder.embed(["oldest"])+ await embedder.embed(["filler-0"])++ assert inner.calls == [["filler-0"]], "the refreshed entry should have survived"+++async def test_a_mutated_hit_does_not_corrupt_the_entry():+ """The hit path must hand back a copy: the miss path returns the inner+ embedder's own list, so only a mutated *hit* can reach the stored entry."""+ inner = _CountingEmbedder()+ embedder = CachingEmbedder(inner)++ await embedder.embed(["q"]) # miss: populates the entry+ hit = await embedder.embed(["q"])+ hit[0][0] = 999.0+ again = await embedder.embed(["q"])++ assert again[0][0] != 999.0+ assert inner.calls == [["q"]], "still served from cache, so the entry is what leaked"+++async def test_dimensions_are_forwarded():+ inner = _CountingEmbedder(dimensions=1536)+ assert CachingEmbedder(inner).dimensions == 1536+++async def test_an_empty_batch_is_delegated_untouched():+ inner = _CountingEmbedder()+ embedder = CachingEmbedder(inner)++ assert await embedder.embed([]) == []+ assert inner.calls == [[]]+ assert embedder._cache == {}+++async def test_a_short_inner_response_is_not_cached():+ """A provider that returns the wrong number of vectors must not have that+ response memoized for the life of the process."""++ class _Empty(_CountingEmbedder):+ async def embed(self, texts):+ self.calls.append(list(texts))+ return []++ inner = _Empty()+ embedder = CachingEmbedder(inner)++ assert await embedder.embed(["q"]) == []+ assert await embedder.embed(["q"]) == []+ assert inner.calls == [["q"], ["q"]], "the bad response was cached"