Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(embedding): harden request-scoped query embedding cache (#4822)

Nick Jamesc82ae5f66485
diff --git a/openviking/models/embedder/base.py b/openviking/models/embedder/base.py
index e6b735e63..d036a1dba 100644
--- a/openviking/models/embedder/base.py
+++ b/openviking/models/embedder/base.py
@@ -7,9 +7,10 @@ import random
import time
import weakref
from abc import ABC, abstractmethod
+from contextlib import contextmanager
from dataclasses import dataclass
from threading import Lock
-from typing import Any, Awaitable, Callable, Dict, List, Optional, TypeVar, Union
+from typing import Any, Awaitable, Callable, Dict, Iterator, List, Optional, TypeVar, Union
from openviking.telemetry import get_current_telemetry
from openviking.utils.embedding_input import (
@@ -41,11 +42,35 @@ query_embed_cache_var: contextvars.ContextVar[Optional[Dict[Any, "asyncio.Task"]
)
+@contextmanager
+def query_embed_cache_scope() -> Iterator[None]:
+ """Install the request-scoped query embedding cache for one request body.
+
+ The scope must wrap the fan-out that spawns the sibling find tasks, not
+ the shared find chokepoint below it: asyncio.gather copies the current
+ context when it wraps each sibling in a task, so a scope installed by the
+ first find to run would stay invisible to its siblings and the dedup would
+ collapse. Callers that fan finds out should wrap that fan-out; callers
+ with no fan-out need no scope.
+ """
+ token = query_embed_cache_var.set({})
+ try:
+ yield
+ finally:
+ query_embed_cache_var.reset(token)
+
+
def _query_cache_key(embedder: "EmbedderBase", embedding_input: "EmbeddingInput") -> tuple:
- """Stable cache key for a prepared embedding input and its embedder."""
+ """Stable cache key for a prepared embedding input and its embedder.
+
+ Keyed by embedder identity rather than ``model_name`` so two embedder
+ instances that happen to share a name (e.g. a future dense/sparse pair)
+ never serve each other's vectors. The embedder is a long-lived process
+ singleton, so its id stays stable for the lifetime of the request scope.
+ """
if isinstance(embedding_input, str):
- return (embedder.model_name, embedding_input)
- return (embedder.model_name, repr(embedding_input))
+ return (id(embedder), embedding_input)
+ return (id(embedder), repr(embedding_input))
# A multimodal embedding input is a list of content parts, e.g.
@@ -142,7 +167,28 @@ async def _embed_from_request_cache(
pending = asyncio.create_task(embedder.embed_async(embedding_input, is_query=True))
cache[key] = pending
try:
- return await pending
+ # Shield the shared task: cancelling a waiter propagates to the future
+ # it is blocked on (Task.cancel cancels its _fut_waiter), so without
+ # the shield a cancelled waiter would kill the embed every sibling is
+ # waiting for and leave the key pointing at a cancelled task.
+ return await asyncio.shield(pending)
+ except asyncio.CancelledError:
+ if pending.cancelled():
+ # The shared embed itself was cancelled: evict the dead entry so
+ # the next find of this text starts a fresh embed instead of
+ # awaiting the cancelled task.
+ cache.pop(key, None)
+ else:
+ # Only this waiter was cancelled; the shared embed lives on. If it
+ # then fails with nobody left waiting on it, evict the stale entry
+ # and consume the exception so asyncio does not log
+ # "Task exception was never retrieved".
+ def _evict_if_failed(done: "asyncio.Task") -> None:
+ if not done.cancelled() and done.exception() is not None:
+ cache.pop(key, None)
+
+ pending.add_done_callback(_evict_if_failed)
+ raise
except Exception:
# A failed embed must not poison the rest of the request.
cache.pop(key, None)
diff --git a/openviking/retrieve/context_assembler/pipeline.py b/openviking/retrieve/context_assembler/pipeline.py
index eb60fad0a..6fce2db04 100644
--- a/openviking/retrieve/context_assembler/pipeline.py
+++ b/openviking/retrieve/context_assembler/pipeline.py
@@ -11,6 +11,7 @@ from __future__ import annotations
from typing import Any, Dict, Optional
+from openviking.models.embedder.base import query_embed_cache_scope
from openviking.retrieve.context_assembler.budget import (
oversized_abstract_needs_body,
per_entry_cap,
@@ -80,19 +81,23 @@ async def assemble_context(
cooled = ledger.cooled_uris() if ledger else set()
excluded = normalize_exclude_uris(params.exclude_uris) | cooled
- candidates, gather_stats = await gather_candidates(
- service=service,
- ctx=ctx,
- queries=queries,
- quotas=quotas,
- limit=params.limit,
- score_threshold=params.score_threshold,
- filter=params.filter,
- image_url=params.image_url,
- peer_scope=params.peer_scope,
- penalties=penalties,
- excluded=excluded,
- )
+ # The fan-out below embeds each planned query once per (query, target)
+ # find, so wrap it in the request-scoped query embedding cache: sibling
+ # finds share the first in-flight embed of each distinct query text.
+ with query_embed_cache_scope():
+ candidates, gather_stats = await gather_candidates(
+ service=service,
+ ctx=ctx,
+ queries=queries,
+ quotas=quotas,
+ limit=params.limit,
+ score_threshold=params.score_threshold,
+ filter=params.filter,
+ image_url=params.image_url,
+ peer_scope=params.peer_scope,
+ penalties=penalties,
+ excluded=excluded,
+ )
# Read only the candidates whose planned tier actually needs a body: with
# the default tiers that is the events bucket, not every hit.
diff --git a/openviking/server/routers/search.py b/openviking/server/routers/search.py
index 22ca0c492..309e050ba 100644
--- a/openviking/server/routers/search.py
+++ b/openviking/server/routers/search.py
@@ -12,7 +12,6 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator
from openviking.core.path_variables import resolve_path_variables
from openviking.core.uri_validation import validate_request_viking_uri
-from openviking.models.embedder.base import query_embed_cache_var
from openviking.pyagfs.exceptions import AGFSClientError, AGFSNotFoundError
from openviking.retrieve.context_assembler import (
CATEGORY_KEYS,
@@ -431,10 +430,6 @@ async def search(
_ctx: RequestContext = Depends(get_request_context),
):
"""Semantic search with optional session context."""
- # Install the request-scoped query embedding cache before any search work
- # spawns tasks: every find fanned out below shares this dict and reuses the
- # first embed of a given query text.
- query_embed_cache_var.set({})
service = get_service()
actual_limit = _resolve_search_limit(request.limit, request.node_limit)
effective_filter = _resolve_search_filter(
diff --git a/openviking/server/routers/skills.py b/openviking/server/routers/skills.py
index 27eefb9b6..63ae2b843 100644
--- a/openviking/server/routers/skills.py
+++ b/openviking/server/routers/skills.py
@@ -20,6 +20,7 @@ from openviking.core.namespace import canonical_user_root
from openviking.core.path_variables import resolve_path_variables
from openviking.core.skill_loader import validate_skill_format
from openviking.core.uri_validation import validate_request_viking_uri
+from openviking.models.embedder.base import query_embed_cache_scope
from openviking.privacy.service import UserPrivacyConfigVersion
from openviking.server.auth import get_request_context
from openviking.server.dependencies import get_service
@@ -599,32 +600,35 @@ async def find_skills(
user_root = f"{canonical_user_root(_ctx)}/skills"
agent_root = "viking://agent/skills"
- user_execution, agent_execution = await asyncio.gather(
- run_operation(
- operation="skills.find",
- telemetry=request.telemetry,
- fn=lambda: service.search.find(
- query=request.query,
- ctx=_ctx,
- target_uri=user_root,
- limit=request.limit,
- score_threshold=request.score_threshold,
- level=request.level,
+ # Both finds embed the same query text, so wrap the fan-out in the
+ # request-scoped cache to reuse the first in-flight embed.
+ with query_embed_cache_scope():
+ user_execution, agent_execution = await asyncio.gather(
+ run_operation(
+ operation="skills.find",
+ telemetry=request.telemetry,
+ fn=lambda: service.search.find(
+ query=request.query,
+ ctx=_ctx,
+ target_uri=user_root,
+ limit=request.limit,
+ score_threshold=request.score_threshold,
+ level=request.level,
+ ),
),
- ),
- run_operation(
- operation="skills.find",
- telemetry=request.telemetry,
- fn=lambda: service.search.find(
- query=request.query,
- ctx=_ctx,
- target_uri=agent_root,
- limit=request.limit,
- score_threshold=request.score_threshold,
- level=request.level,
+ run_operation(
+ operation="skills.find",
+ telemetry=request.telemetry,
+ fn=lambda: service.search.find(
+ query=request.query,
+ ctx=_ctx,
+ target_uri=agent_root,
+ limit=request.limit,
+ score_threshold=request.score_threshold,
+ level=request.level,
+ ),
),
- ),
- )
+ )
user_result = user_execution.result
user_result_dict = (
diff --git a/tests/models/test_query_embedding_cache.py b/tests/models/test_query_embedding_cache.py
index 4d9459000..ff2c44ecb 100644
--- a/tests/models/test_query_embedding_cache.py
+++ b/tests/models/test_query_embedding_cache.py
@@ -5,9 +5,11 @@ from __future__ import annotations
import asyncio
from typing import List
+import pytest
+
from openviking.models.embedder.base import (
- EmbedResult,
EmbedderBase,
+ EmbedResult,
embed_compat,
query_embed_cache_var,
)
@@ -16,10 +18,11 @@ from openviking.models.embedder.base import (
class CountingEmbedder(EmbedderBase):
"""Fake embedder that records every async embed and can be told to fail."""
- def __init__(self, model_name: str = "fake-model"):
+ def __init__(self, model_name: str = "fake-model", delay: float = 0):
super().__init__(model_name)
self.calls: List[str] = []
self.fail_next = 0
+ self.delay = delay
def embed(self, content, is_query: bool = False) -> EmbedResult:
self.calls.append(str(content))
@@ -27,7 +30,10 @@ class CountingEmbedder(EmbedderBase):
async def embed_async(self, content, is_query: bool = False) -> EmbedResult:
self.calls.append(str(content))
- await asyncio.sleep(0)
+ if self.delay:
+ await asyncio.sleep(self.delay)
+ else:
+ await asyncio.sleep(0)
if self.fail_next > 0:
self.fail_next -= 1
raise RuntimeError("embed failed")
@@ -108,3 +114,100 @@ async def test_embed_compat_retries_after_a_failed_embed():
result = await embed_compat(embedder, "hello", is_query=True)
assert result.dense_vector == [1.0]
assert embedder.calls == ["hello", "hello"]
+
+
+async def test_query_embed_cache_scope_resets_after_exit():
+ from openviking.models.embedder.base import query_embed_cache_scope
+
+ with query_embed_cache_scope():
+ assert query_embed_cache_var.get() is not None
+ assert query_embed_cache_var.get() is None
+
+
+async def test_query_embed_cache_scope_resets_on_exception():
+ from openviking.models.embedder.base import query_embed_cache_scope
+
+ with pytest.raises(RuntimeError):
+ with query_embed_cache_scope():
+ raise RuntimeError("boom")
+ assert query_embed_cache_var.get() is None
+
+
+async def test_embedders_with_same_model_name_do_not_share_entries():
+ # Keying by embedder identity keeps two embedders that happen to share a
+ # model_name from serving each other's vectors.
+ first = CountingEmbedder(model_name="shared-name")
+ second = CountingEmbedder(model_name="shared-name")
+ query_embed_cache_var.set({})
+
+ await embed_compat(first, "same-text", is_query=True)
+ await embed_compat(second, "same-text", is_query=True)
+
+ assert first.calls == ["same-text"]
+ assert second.calls == ["same-text"]
+
+
+async def test_waiter_cancellation_leaves_shared_embed_running():
+ # Cancelling a waiter must not cancel the shared in-flight embed (shield),
+ # and the cache entry must stay usable for later waiters.
+ embedder = CountingEmbedder(delay=0.1)
+ query_embed_cache_var.set({})
+
+ waiter = asyncio.create_task(embed_compat(embedder, "shared", is_query=True))
+ await asyncio.sleep(0.01) # let the waiter create and await the shared task
+ waiter.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+
+ # The shared embed survived the waiter cancellation and completed.
+ await asyncio.sleep(0.15)
+ assert embedder.calls == ["shared"]
+
+ result = await embed_compat(embedder, "shared", is_query=True)
+ assert result.dense_vector == [1.0]
+ assert embedder.calls == ["shared"] # served from cache, no second embed
+
+
+async def test_cancelled_shared_embed_is_evicted_and_retried():
+ # When the shared task itself is cancelled, waiters must observe the
+ # cancellation, the poisoned key must be evicted, and the next embed of the
+ # same text must start a fresh task instead of awaiting the dead one.
+ embedder = CountingEmbedder(delay=0.1)
+ query_embed_cache_var.set({})
+
+ waiter = asyncio.create_task(embed_compat(embedder, "shared", is_query=True))
+ await asyncio.sleep(0.01) # let the waiter create and await the shared task
+ cache = query_embed_cache_var.get()
+ assert cache is not None
+ next(iter(cache.values())).cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+ assert cache == {}
+
+ result = await embed_compat(embedder, "shared", is_query=True)
+ assert result.dense_vector == [1.0]
+ assert embedder.calls == ["shared", "shared"]
+
+
+async def test_waiter_cancelled_then_shared_embed_failure_evicts_entry():
+ # The last waiter is cancelled while the shared embed still runs; when that
+ # orphaned embed then fails, the stale entry must be evicted so the next
+ # embed of the same text starts fresh instead of replaying the old failure.
+ embedder = CountingEmbedder(delay=0.1)
+ embedder.fail_next = 1
+ query_embed_cache_var.set({})
+
+ waiter = asyncio.create_task(embed_compat(embedder, "shared", is_query=True))
+ await asyncio.sleep(0.01) # let the waiter create and await the shared task
+ waiter.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await waiter
+
+ # Let the orphaned shared embed run to failure; the done callback must
+ # have evicted the failed entry.
+ await asyncio.sleep(0.2)
+ assert query_embed_cache_var.get() == {}
+
+ result = await embed_compat(embedder, "shared", is_query=True)
+ assert result.dense_vector == [1.0]
+ assert embedder.calls == ["shared", "shared"]