Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

perf(mcp): cache query embeddings so a repeated search skips the provider (#2189)

Raghav Chamadiya5ffc478120ca
diff --git a/packages/core/src/repowise/core/providers/embedding/caching.py b/packages/core/src/repowise/core/providers/embedding/caching.py
new file mode 100644
index 000000000..6220559a6
--- /dev/null
+++ b/packages/core/src/repowise/core/providers/embedding/caching.py
@@ -0,0 +1,66 @@
+"""Memoize query embeddings so a repeated search does not pay the provider twice.
+
+Every ``search_codebase`` call embeds its query text through the configured
+provider — a network round trip, measured at ~215ms against OpenAI, on a path
+where the vector lookup itself costs ~35ms. Agents repeat queries within a
+session, and each repeat paid that round trip again.
+
+Wrap the embedder rather than caching inside one provider or one store: the
+query paths reach the embedder through three different store methods
+(``search``, ``search_many``, ``embed_texts``), and there are five providers
+with the same shape, so this is the one layer common to all of them.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections import OrderedDict
+from typing import Any
+
+#: Entries kept per wrapped embedder. A 1536-dimension vector costs roughly
+#: 40KB live as a Python list, so this bounds one embedder at a few megabytes.
+MAX_ENTRIES = 256
+
+
+class CachingEmbedder:
+ """An :class:`~repowise.core.providers.embedding.base.Embedder` that memoizes single texts.
+
+ A cached vector is a pure function of (embedder configuration, text), so a
+ hit is exactly as correct as a fresh call and never goes stale. The cache
+ lives on the instance, so the configuration behind it is fixed for the life
+ of every entry and the text alone identifies a vector.
+
+ The gate is batch size, not caller: only single-text calls are cached. That
+ is a proxy rather than a rule — a single-document write is cached too, and
+ harmlessly, since the vector is the same either way. What it reliably keeps
+ out is the bulk path, where hundreds of corpus strings that are never
+ queried again would evict the handful of real queries this exists for.
+ """
+
+ def __init__(self, inner: Any) -> None:
+ self._inner = inner
+ self._cache: OrderedDict[str, list[float]] = OrderedDict()
+
+ @property
+ def dimensions(self) -> int:
+ return self._inner.dimensions
+
+ async def embed(self, texts: list[str]) -> list[list[float]]:
+ if len(texts) != 1:
+ return await self._inner.embed(texts)
+
+ # Hashed rather than stored: a query reaches 30_000 characters upstream,
+ # and holding the raw text would cost more than the vector it caches.
+ key = hashlib.sha256(texts[0].encode()).hexdigest()
+ cached = self._cache.get(key)
+ if cached is not None:
+ self._cache.move_to_end(key)
+ # Copy: a caller that mutates its vector must not edit the entry.
+ return [list(cached)]
+
+ vectors = await self._inner.embed(texts)
+ if len(vectors) == 1:
+ self._cache[key] = list(vectors[0])
+ while len(self._cache) > MAX_ENTRIES:
+ self._cache.popitem(last=False)
+ return vectors
diff --git a/packages/server/src/repowise/server/mcp_server/_server.py b/packages/server/src/repowise/server/mcp_server/_server.py
index 10a584512..de5acebd0 100644
--- a/packages/server/src/repowise/server/mcp_server/_server.py
+++ b/packages/server/src/repowise/server/mcp_server/_server.py
@@ -24,7 +24,8 @@ from repowise.core.persistence.database import (
from repowise.core.persistence.search import FullTextSearch
from repowise.core.persistence.vector_store import InMemoryVectorStore
from repowise.core.platform.telemetry import GROUP_LEAF_TYPES_ATTR
-from repowise.core.providers.embedding.base import KeylessEmbedder
+from repowise.core.providers.embedding.base import KeylessEmbedder, is_semantic_embedder
+from repowise.core.providers.embedding.caching import CachingEmbedder
from repowise.server.mcp_server import _state
_log = __import__("logging").getLogger("repowise.mcp")
@@ -289,6 +290,23 @@ def _resolve_embedder():
return KeylessEmbedder()
+def _query_embedder():
+ """The embedder this server answers queries with.
+
+ Wrapped so a repeated query does not pay the provider round trip again.
+
+ Only this process wraps: a CLI invocation is one-shot and serves a single
+ query, so a per-process cache could never hit there. The HTTP server
+ (``server/app.py``) is long-lived and would benefit, and is left unwrapped
+ only to keep this change to the surface the cost was measured on.
+
+ Keyless is left bare — :func:`is_semantic_embedder` identifies it by type
+ to switch the vector leg off, and a wrapper would defeat that.
+ """
+ embedder = _resolve_embedder()
+ return CachingEmbedder(embedder) if is_semantic_embedder(embedder) else embedder
+
+
#: How often the running server re-reads release currency. The PyPI fetch
#: itself is TTL-cached on disk for a day and shared with the CLI advisory, so
#: this bounds only the in-process refresh, never the network.
@@ -383,7 +401,7 @@ async def _load_vector_stores(repo_path: str | None) -> None:
import asyncio as _asyncio
try:
- embedder = _resolve_embedder()
+ embedder = _query_embedder()
vector_store: Any = InMemoryVectorStore(embedder=embedder)
try:
@@ -510,7 +528,7 @@ async def _lifespan(server: FastMCP):
registry = RepoRegistry(
workspace_root=ws_root,
ws_config=ws_config,
- embedder_factory=lambda: _resolve_embedder(),
+ embedder_factory=_query_embedder,
)
# Eagerly load the default repo so tools work immediately. A failure
diff --git a/tests/unit/server/mcp/test_embedder_resolution.py b/tests/unit/server/mcp/test_embedder_resolution.py
index 5173a825a..6043d2368 100644
--- a/tests/unit/server/mcp/test_embedder_resolution.py
+++ b/tests/unit/server/mcp/test_embedder_resolution.py
@@ -16,7 +16,12 @@ from pathlib import Path
import pytest
-from repowise.core.providers.embedding.base import MockEmbedder
+from repowise.core.providers.embedding.base import (
+ KeylessEmbedder,
+ MockEmbedder,
+ is_semantic_embedder,
+)
+from repowise.core.providers.embedding.caching import CachingEmbedder
from repowise.server.mcp_server import _server, _state
from repowise.server.mcp_server._meta import build_meta
@@ -330,3 +335,39 @@ def test_build_meta_omits_degraded_when_embedder_unresolved(monkeypatch):
meta = build_meta(timing_ms=1.0)
assert "embedder_degraded" not in meta
assert "semantic_search" not in meta
+
+
+class TestQueryEmbedderWrapping:
+ """``_query_embedder`` is what the server's stores actually embed with.
+
+ It memoizes repeated queries, but must never wrap a keyless embedder:
+ ``is_semantic_embedder`` identifies that one by type in order to switch the
+ vector leg off, and a wrapper would report as semantic and silently
+ reactivate a leg whose vectors carry no signal.
+ """
+
+ def test_configured_embedder_is_wrapped_for_caching(self, monkeypatch, tmp_path):
+ monkeypatch.setenv("REPOWISE_EMBEDDER", "openai")
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+
+ embedder = _server._query_embedder()
+
+ assert isinstance(embedder, CachingEmbedder)
+ assert is_semantic_embedder(embedder)
+
+ def test_unconfigured_keyless_is_left_unwrapped(self):
+ embedder = _server._query_embedder()
+
+ assert isinstance(embedder, KeylessEmbedder)
+ assert not is_semantic_embedder(embedder), "the vector leg must stay off"
+
+ def test_degraded_fallback_is_left_unwrapped(self, monkeypatch):
+ """A configured embedder that cannot initialise falls back to keyless —
+ wrapping that would hide it from the vector-leg check."""
+ monkeypatch.setenv("REPOWISE_EMBEDDER", "openai")
+
+ embedder = _server._query_embedder()
+
+ assert _state._embedder_status["degraded"] is True
+ assert isinstance(embedder, KeylessEmbedder)
+ assert not is_semantic_embedder(embedder)
diff --git a/tests/unit/test_persistence/test_caching_embedder.py b/tests/unit/test_persistence/test_caching_embedder.py
new file mode 100644
index 000000000..e1ca5e2a6
--- /dev/null
+++ b/tests/unit/test_persistence/test_caching_embedder.py
@@ -0,0 +1,138 @@
+"""Tests for the query-embedding cache wrapper.
+
+Every ``search_codebase`` embeds its query through the provider — a network
+round trip on a path where the vector lookup itself is an order of magnitude
+cheaper. These pin that a repeat is served locally, and that the cache cannot
+hand back a vector from a different embedder or a different width.
+"""
+
+from __future__ import annotations
+
+from repowise.core.providers.embedding.caching import MAX_ENTRIES, CachingEmbedder
+
+
+class _CountingEmbedder:
+ """Records every batch it is asked to embed."""
+
+ def __init__(self, dimensions: int = 4, seed: float = 1.0) -> None:
+ self._dimensions = dimensions
+ self._seed = seed
+ self.calls: list[list[str]] = []
+
+ @property
+ def dimensions(self) -> int:
+ return self._dimensions
+
+ async def embed(self, texts: list[str]) -> list[list[float]]:
+ self.calls.append(list(texts))
+ return [[self._seed * (i + 1)] * self._dimensions for i, _ in enumerate(texts)]
+
+
+async def test_repeated_query_is_served_from_cache():
+ inner = _CountingEmbedder()
+ embedder = CachingEmbedder(inner)
+
+ first = await embedder.embed(["auth service"])
+ second = await embedder.embed(["auth service"])
+
+ assert first == second
+ assert inner.calls == [["auth service"]], "the provider was asked twice"
+
+
+async def test_distinct_queries_each_reach_the_provider():
+ inner = _CountingEmbedder()
+ embedder = CachingEmbedder(inner)
+
+ await embedder.embed(["one"])
+ await embedder.embed(["two"])
+
+ assert inner.calls == [["one"], ["two"]]
+
+
+async def test_multi_text_batches_are_not_cached():
+ """Bulk calls are indexing or decision matching, never repeated queries."""
+ inner = _CountingEmbedder()
+ embedder = CachingEmbedder(inner)
+
+ await embedder.embed(["a", "b"])
+ await embedder.embed(["a", "b"])
+
+ assert inner.calls == [["a", "b"], ["a", "b"]]
+
+
+async def test_a_bulk_call_does_not_evict_or_serve_queries():
+ inner = _CountingEmbedder()
+ embedder = CachingEmbedder(inner)
+
+ await embedder.embed(["query"])
+ await embedder.embed(["query", "other"])
+ await embedder.embed(["query"])
+
+ # The single-text query was embedded once; the batch neither used nor
+ # displaced its entry.
+ assert inner.calls == [["query"], ["query", "other"]]
+
+
+async def test_cache_is_bounded_and_evicts_least_recently_used():
+ """A per-query entry that is never evicted is a leak in a long-lived server."""
+ inner = _CountingEmbedder()
+ embedder = CachingEmbedder(inner)
+
+ await embedder.embed(["oldest"])
+ for i in range(MAX_ENTRIES - 1):
+ await embedder.embed([f"filler-{i}"])
+ await embedder.embed(["oldest"]) # refresh, so a filler is now least recent
+ await embedder.embed(["overflow"]) # pushes past the cap
+
+ assert len(embedder._cache) == MAX_ENTRIES
+ inner.calls.clear()
+ await embedder.embed(["oldest"])
+ await embedder.embed(["filler-0"])
+
+ assert inner.calls == [["filler-0"]], "the refreshed entry should have survived"
+
+
+async def test_a_mutated_hit_does_not_corrupt_the_entry():
+ """The hit path must hand back a copy: the miss path returns the inner
+ embedder's own list, so only a mutated *hit* can reach the stored entry."""
+ inner = _CountingEmbedder()
+ embedder = CachingEmbedder(inner)
+
+ await embedder.embed(["q"]) # miss: populates the entry
+ hit = await embedder.embed(["q"])
+ hit[0][0] = 999.0
+ again = await embedder.embed(["q"])
+
+ assert again[0][0] != 999.0
+ assert inner.calls == [["q"]], "still served from cache, so the entry is what leaked"
+
+
+async def test_dimensions_are_forwarded():
+ inner = _CountingEmbedder(dimensions=1536)
+ assert CachingEmbedder(inner).dimensions == 1536
+
+
+async def test_an_empty_batch_is_delegated_untouched():
+ inner = _CountingEmbedder()
+ embedder = CachingEmbedder(inner)
+
+ assert await embedder.embed([]) == []
+ assert inner.calls == [[]]
+ assert embedder._cache == {}
+
+
+async def test_a_short_inner_response_is_not_cached():
+ """A provider that returns the wrong number of vectors must not have that
+ response memoized for the life of the process."""
+
+ class _Empty(_CountingEmbedder):
+ async def embed(self, texts):
+ self.calls.append(list(texts))
+ return []
+
+ inner = _Empty()
+ embedder = CachingEmbedder(inner)
+
+ assert await embedder.embed(["q"]) == []
+ assert await embedder.embed(["q"]) == []
+ assert inner.calls == [["q"], ["q"]], "the bad response was cached"