Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(embedding): harden request-scoped query embedding cache (#4822)
Nick Jamesc82ae5f66485
diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.pyindex e6b735e63..d036a1dba 100644--- a/openviking/models/embedder/base.py+++ b/openviking/models/embedder/base.py@@ -7,9 +7,10 @@ import randomimport timeimport weakreffrom abc import ABC, abstractmethod+from contextlib import contextmanagerfrom dataclasses import dataclassfrom threading import Lock-from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar, Union+from typing import Any, Awaitable, Callable, Dict, Iterator, List, Optional, TypeVar, Unionfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.embedding_input import (@@ -41,11 +42,35 @@ query_embed_cache_var: contextvars.ContextVar[Optional[Dict[Any, "asyncio.Task"])+@contextmanager+def query_embed_cache_scope() -> Iterator[None]:+ """Install the request-scoped query embedding cache for one request body.++ The scope must wrap the fan-out that spawns the sibling find tasks, not+ the shared find chokepoint below it: asyncio.gather copies the current+ context when it wraps each sibling in a task, so a scope installed by the+ first find to run would stay invisible to its siblings and the dedup would+ collapse. Callers that fan finds out should wrap that fan-out; callers+ with no fan-out need no scope.+ """+ token = query_embed_cache_var.set({})+ try:+ yield+ finally:+ query_embed_cache_var.reset(token)++def _query_cache_key(embedder: "EmbedderBase", embedding_input: "EmbeddingInput") -> tuple:- """Stable cache key for a prepared embedding input and its embedder."""+ """Stable cache key for a prepared embedding input and its embedder.++ Keyed by embedder identity rather than ``model_name`` so two embedder+ instances that happen to share a name (e.g. a future dense/sparse pair)+ never serve each other's vectors. The embedder is a long-lived process+ singleton, so its id stays stable for the lifetime of the request scope.+ """if isinstance(embedding_input, str):- return (embedder.model_name, embedding_input)- return (embedder.model_name, repr(embedding_input))+ return (id(embedder), embedding_input)+ return (id(embedder), repr(embedding_input))# A multimodal embedding input is a list of content parts, e.g.@@ -142,7 +167,28 @@ async def _embed_from_request_cache(pending = asyncio.create_task(embedder.embed_async(embedding_input, is_query=True))cache[key] = pendingtry:- return await pending+ # Shield the shared task: cancelling a waiter propagates to the future+ # it is blocked on (Task.cancel cancels its _fut_waiter), so without+ # the shield a cancelled waiter would kill the embed every sibling is+ # waiting for and leave the key pointing at a cancelled task.+ return await asyncio.shield(pending)+ except asyncio.CancelledError:+ if pending.cancelled():+ # The shared embed itself was cancelled: evict the dead entry so+ # the next find of this text starts a fresh embed instead of+ # awaiting the cancelled task.+ cache.pop(key, None)+ else:+ # Only this waiter was cancelled; the shared embed lives on. If it+ # then fails with nobody left waiting on it, evict the stale entry+ # and consume the exception so asyncio does not log+ # "Task exception was never retrieved".+ def _evict_if_failed(done: "asyncio.Task") -> None:+ if not done.cancelled() and done.exception() is not None:+ cache.pop(key, None)++ pending.add_done_callback(_evict_if_failed)+ raiseexcept Exception:# A failed embed must not poison the rest of the request.cache.pop(key, None)diff --git a/openviking/retrieve/context_assembler/pipeline.py b/openviking/retrieve/context_assembler/pipeline.pyindex eb60fad0a..6fce2db04 100644--- a/openviking/retrieve/context_assembler/pipeline.py+++ b/openviking/retrieve/context_assembler/pipeline.py@@ -11,6 +11,7 @@ from __future__ import annotationsfrom typing import Any, Dict, Optional+from openviking.models.embedder.base import query_embed_cache_scopefrom openviking.retrieve.context_assembler.budget import (oversized_abstract_needs_body,per_entry_cap,@@ -80,19 +81,23 @@ async def assemble_context(cooled = ledger.cooled_uris() if ledger else set()excluded = normalize_exclude_uris(params.exclude_uris) | cooled- candidates, gather_stats = await gather_candidates(- service=service,- ctx=ctx,- queries=queries,- quotas=quotas,- limit=params.limit,- score_threshold=params.score_threshold,- filter=params.filter,- image_url=params.image_url,- peer_scope=params.peer_scope,- penalties=penalties,- excluded=excluded,- )+ # The fan-out below embeds each planned query once per (query, target)+ # find, so wrap it in the request-scoped query embedding cache: sibling+ # finds share the first in-flight embed of each distinct query text.+ with query_embed_cache_scope():+ candidates, gather_stats = await gather_candidates(+ service=service,+ ctx=ctx,+ queries=queries,+ quotas=quotas,+ limit=params.limit,+ score_threshold=params.score_threshold,+ filter=params.filter,+ image_url=params.image_url,+ peer_scope=params.peer_scope,+ penalties=penalties,+ excluded=excluded,+ )# Read only the candidates whose planned tier actually needs a body: with# the default tiers that is the events bucket, not every hit.diff --git a/openviking/server/routers/search.py b/openviking/server/routers/search.pyindex 22ca0c492..309e050ba 100644--- a/openviking/server/routers/search.py+++ b/openviking/server/routers/search.py@@ -12,7 +12,6 @@ from pydantic import BaseModel, ConfigDict, Field, model_validatorfrom openviking.core.path_variables import resolve_path_variablesfrom openviking.core.uri_validation import validate_request_viking_uri-from openviking.models.embedder.base import query_embed_cache_varfrom openviking.pyagfs.exceptions import AGFSClientError, AGFSNotFoundErrorfrom openviking.retrieve.context_assembler import (CATEGORY_KEYS,@@ -431,10 +430,6 @@ async def search(_ctx: RequestContext = Depends(get_request_context),):"""Semantic search with optional session context."""- # Install the request-scoped query embedding cache before any search work- # spawns tasks: every find fanned out below shares this dict and reuses the- # first embed of a given query text.- query_embed_cache_var.set({})service = get_service()actual_limit = _resolve_search_limit(request.limit, request.node_limit)effective_filter = _resolve_search_filter(diff --git a/openviking/server/routers/skills.py b/openviking/server/routers/skills.pyindex 27eefb9b6..63ae2b843 100644--- a/openviking/server/routers/skills.py+++ b/openviking/server/routers/skills.py@@ -20,6 +20,7 @@ from openviking.core.namespace import canonical_user_rootfrom openviking.core.path_variables import resolve_path_variablesfrom openviking.core.skill_loader import validate_skill_formatfrom openviking.core.uri_validation import validate_request_viking_uri+from openviking.models.embedder.base import query_embed_cache_scopefrom openviking.privacy.service import UserPrivacyConfigVersionfrom openviking.server.auth import get_request_contextfrom openviking.server.dependencies import get_service@@ -599,32 +600,35 @@ async def find_skills(user_root = f"{canonical_user_root(_ctx)}/skills"agent_root = "viking://agent/skills"- user_execution, agent_execution = await asyncio.gather(- run_operation(- operation="skills.find",- telemetry=request.telemetry,- fn=lambda: service.search.find(- query=request.query,- ctx=_ctx,- target_uri=user_root,- limit=request.limit,- score_threshold=request.score_threshold,- level=request.level,+ # Both finds embed the same query text, so wrap the fan-out in the+ # request-scoped cache to reuse the first in-flight embed.+ with query_embed_cache_scope():+ user_execution, agent_execution = await asyncio.gather(+ run_operation(+ operation="skills.find",+ telemetry=request.telemetry,+ fn=lambda: service.search.find(+ query=request.query,+ ctx=_ctx,+ target_uri=user_root,+ limit=request.limit,+ score_threshold=request.score_threshold,+ level=request.level,+ ),),- ),- run_operation(- operation="skills.find",- telemetry=request.telemetry,- fn=lambda: service.search.find(- query=request.query,- ctx=_ctx,- target_uri=agent_root,- limit=request.limit,- score_threshold=request.score_threshold,- level=request.level,+ run_operation(+ operation="skills.find",+ telemetry=request.telemetry,+ fn=lambda: service.search.find(+ query=request.query,+ ctx=_ctx,+ target_uri=agent_root,+ limit=request.limit,+ score_threshold=request.score_threshold,+ level=request.level,+ ),),- ),- )+ )user_result = user_execution.resultuser_result_dict = (diff --git a/tests/models/test_query_embedding_cache.py b/tests/models/test_query_embedding_cache.pyindex 4d9459000..ff2c44ecb 100644--- a/tests/models/test_query_embedding_cache.py+++ b/tests/models/test_query_embedding_cache.py@@ -5,9 +5,11 @@ from __future__ import annotationsimport asynciofrom typing import List+import pytest+from openviking.models.embedder.base import (- EmbedResult,EmbedderBase,+ EmbedResult,embed_compat,query_embed_cache_var,)@@ -16,10 +18,11 @@ from openviking.models.embedder.base import (class CountingEmbedder(EmbedderBase):"""Fake embedder that records every async embed and can be told to fail."""- def __init__(self, model_name: str = "fake-model"):+ def __init__(self, model_name: str = "fake-model", delay: float = 0):super().__init__(model_name)self.calls: List[str] = []self.fail_next = 0+ self.delay = delaydef embed(self, content, is_query: bool = False) -> EmbedResult:self.calls.append(str(content))@@ -27,7 +30,10 @@ class CountingEmbedder(EmbedderBase):async def embed_async(self, content, is_query: bool = False) -> EmbedResult:self.calls.append(str(content))- await asyncio.sleep(0)+ if self.delay:+ await asyncio.sleep(self.delay)+ else:+ await asyncio.sleep(0)if self.fail_next > 0:self.fail_next -= 1raise RuntimeError("embed failed")@@ -108,3 +114,100 @@ async def test_embed_compat_retries_after_a_failed_embed():result = await embed_compat(embedder, "hello", is_query=True)assert result.dense_vector == [1.0]assert embedder.calls == ["hello", "hello"]+++async def test_query_embed_cache_scope_resets_after_exit():+ from openviking.models.embedder.base import query_embed_cache_scope++ with query_embed_cache_scope():+ assert query_embed_cache_var.get() is not None+ assert query_embed_cache_var.get() is None+++async def test_query_embed_cache_scope_resets_on_exception():+ from openviking.models.embedder.base import query_embed_cache_scope++ with pytest.raises(RuntimeError):+ with query_embed_cache_scope():+ raise RuntimeError("boom")+ assert query_embed_cache_var.get() is None+++async def test_embedders_with_same_model_name_do_not_share_entries():+ # Keying by embedder identity keeps two embedders that happen to share a+ # model_name from serving each other's vectors.+ first = CountingEmbedder(model_name="shared-name")+ second = CountingEmbedder(model_name="shared-name")+ query_embed_cache_var.set({})++ await embed_compat(first, "same-text", is_query=True)+ await embed_compat(second, "same-text", is_query=True)++ assert first.calls == ["same-text"]+ assert second.calls == ["same-text"]+++async def test_waiter_cancellation_leaves_shared_embed_running():+ # Cancelling a waiter must not cancel the shared in-flight embed (shield),+ # and the cache entry must stay usable for later waiters.+ embedder = CountingEmbedder(delay=0.1)+ query_embed_cache_var.set({})++ waiter = asyncio.create_task(embed_compat(embedder, "shared", is_query=True))+ await asyncio.sleep(0.01) # let the waiter create and await the shared task+ waiter.cancel()+ with pytest.raises(asyncio.CancelledError):+ await waiter++ # The shared embed survived the waiter cancellation and completed.+ await asyncio.sleep(0.15)+ assert embedder.calls == ["shared"]++ result = await embed_compat(embedder, "shared", is_query=True)+ assert result.dense_vector == [1.0]+ assert embedder.calls == ["shared"] # served from cache, no second embed+++async def test_cancelled_shared_embed_is_evicted_and_retried():+ # When the shared task itself is cancelled, waiters must observe the+ # cancellation, the poisoned key must be evicted, and the next embed of the+ # same text must start a fresh task instead of awaiting the dead one.+ embedder = CountingEmbedder(delay=0.1)+ query_embed_cache_var.set({})++ waiter = asyncio.create_task(embed_compat(embedder, "shared", is_query=True))+ await asyncio.sleep(0.01) # let the waiter create and await the shared task+ cache = query_embed_cache_var.get()+ assert cache is not None+ next(iter(cache.values())).cancel()+ with pytest.raises(asyncio.CancelledError):+ await waiter+ assert cache == {}++ result = await embed_compat(embedder, "shared", is_query=True)+ assert result.dense_vector == [1.0]+ assert embedder.calls == ["shared", "shared"]+++async def test_waiter_cancelled_then_shared_embed_failure_evicts_entry():+ # The last waiter is cancelled while the shared embed still runs; when that+ # orphaned embed then fails, the stale entry must be evicted so the next+ # embed of the same text starts fresh instead of replaying the old failure.+ embedder = CountingEmbedder(delay=0.1)+ embedder.fail_next = 1+ query_embed_cache_var.set({})++ waiter = asyncio.create_task(embed_compat(embedder, "shared", is_query=True))+ await asyncio.sleep(0.01) # let the waiter create and await the shared task+ waiter.cancel()+ with pytest.raises(asyncio.CancelledError):+ await waiter++ # Let the orphaned shared embed run to failure; the done callback must+ # have evicted the failed entry.+ await asyncio.sleep(0.2)+ assert query_embed_cache_var.get() == {}++ result = await embed_compat(embedder, "shared", is_query=True)+ assert result.dense_vector == [1.0]+ assert embedder.calls == ["shared", "shared"]