Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(hermes-plugin): report query recall timeouts (#5391)
z1gonec646203e960
diff --git a/examples/hermes-plugin/__init__.py b/examples/hermes-plugin/__init__.pyindex b2bdfdc9f..488d7c4bb 100644--- a/examples/hermes-plugin/__init__.py+++ b/examples/hermes-plugin/__init__.py@@ -268,6 +268,17 @@ def _get_httpx():return None+def _is_timeout_error(error: BaseException) -> bool:+ """Identify socket and HTTP transport timeouts in query recall."""+ if isinstance(error, TimeoutError):+ return True+ try:+ from httpx import TimeoutException+ except ImportError:+ return False+ return isinstance(error, TimeoutException)++class _VikingClient:"""Thin HTTP client for the OpenViking REST API (httpx, no SDK dependency)."""@@ -500,13 +511,17 @@ _TOOL_STATUS_ERROR_ALIASES = {"error", "failed", "failure"}_TOOL_STATUS_COMPLETED_ALIASES = {"completed", "complete", "success", "succeeded"}-def _resolve_user_space(client, *, timeout: Optional[float] = None) -> Optional[str]:+def _resolve_user_space(client, *, timeout: Optional[float] = None,+ raise_on_timeout: bool = False) -> Optional[str]:"""Server-asserted current user for explicit-uid URIs; ``None`` when the probe fails orreports no user. Callers may fall back to a configured value for that one operation but- must not cache an unverified identity — a later probe can succeed."""+ must not cache an unverified identity — a later probe can succeed.+ Query recall can propagate timeouts to its bounded warning handler."""try:status = client.get("/api/v1/system/status", **({"timeout": timeout} if timeout is not None else {}))- except Exception:+ except Exception as exc:+ if raise_on_timeout and _is_timeout_error(exc):+ raiselogger.debug("OpenViking user-space probe failed; using configured fallback", exc_info=True)return Nonereturn str(((status or {}).get("result") or {}).get("user") or "").strip() or None@@ -1808,6 +1823,7 @@ class OpenVikingMemoryProvider(MemoryProvider):if client is None:return ""+ cfg = Nonetry:cfg = self._recall_config()deadline = time.monotonic() + cfg["timeout_seconds"]@@ -1820,7 +1836,10 @@ class OpenVikingMemoryProvider(MemoryProvider):if scope == "peer":# Explicit roots also constrain fallback searches when there is# no sender. An actor-less user-root search includes all peers.- user = _resolve_user_space(client, timeout=self._remaining_recall_timeout(deadline, cfg["request_timeout_seconds"]))+ user = _resolve_user_space(+ client, timeout=self._remaining_recall_timeout(deadline, cfg["request_timeout_seconds"]),+ raise_on_timeout=True,+ )if not user:return ""user_root = f"viking://user/{user}"@@ -1896,7 +1915,15 @@ class OpenVikingMemoryProvider(MemoryProvider):deadline=deadline, request_timeout=cfg["request_timeout_seconds"], full_read_limit=cfg["full_read_limit"],))except Exception as e:- logger.debug("OpenViking context search failed: %s", e)+ # A timeout leaves query recall empty. Report only local, bounded+ # diagnostics; exception text can contain query or identity data.+ if cfg is not None and _is_timeout_error(e):+ logger.warning(+ "OpenViking recall timed out (%s; budget_s=%s request_s=%s); no query context injected",+ type(e).__name__, cfg["timeout_seconds"], cfg["request_timeout_seconds"],+ )+ else:+ logger.debug("OpenViking context search failed: %s", e)return ""# -- typed settings ------------------------------------------------------diff --git a/examples/hermes-plugin/tests/test_recall_timeout.py b/examples/hermes-plugin/tests/test_recall_timeout.pynew file mode 100644index 000000000..95a8e9439--- /dev/null+++ b/examples/hermes-plugin/tests/test_recall_timeout.py@@ -0,0 +1,195 @@+"""Recall timeout diagnostics through the installed external provider."""++import json+import logging+from types import SimpleNamespace++import httpx+import pytest+++def recall_provider(external_provider, monkeypatch):+ home, provider, module, _ = external_provider("recall-timeout")+ (home / "config.yaml").write_text(+ "memory:\n provider: openviking\n openviking:\n"+ " endpoint: http://openviking.test\n"+ " use_ovcli_config: false\n"+ " recall_timeout_seconds: 1.5\n"+ " recall_request_timeout_seconds: 0.75\n"+ " recall_prefer_abstract: true\n",+ encoding="utf-8",+ )+ requests = []+ behavior = {"error": None, "fallback": False, "respond": None, "home": home}++ def handle(request):+ requests.append(request.url.path)+ if behavior["respond"] is not None:+ return behavior["respond"](request)+ if behavior["error"] is not None and not (+ behavior["fallback"] and request.url.path == "/api/v1/search/find"+ ):+ raise behavior["error"]("private query and identity in exception")+ return httpx.Response(+ 200,+ json={"result": {"memories": [{+ "uri": "viking://user/private-identity/memories/decision.md",+ "abstract": "The decision is retained.",+ "score": 0.9,+ }]}},+ )++ with httpx.Client(transport=httpx.MockTransport(handle)) as client:+ monkeypatch.setattr(httpx, "get", client.get)+ monkeypatch.setattr(httpx, "post", client.post)+ monkeypatch.setattr(module, "_classify_runtime_openviking_health", lambda *_: ("healthy", ""))+ provider.initialize(+ "recall-session", hermes_home=str(home), platform="telegram", user_id="private-sender"+ )+ yield provider, module, requests, behavior++[email protected]("error", [+ httpx.ReadTimeout, httpx.ConnectTimeout, httpx.WriteTimeout,+ httpx.PoolTimeout, TimeoutError,+])+def test_timeout_warns_without_private_text_or_extra_requests(recall_provider, caplog, error):+ provider, _module, requests, behavior = recall_provider+ behavior["error"] = error+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):+ assert provider._search_prefetch_context(+ "private query", client=provider._client+ ) == ""++ warnings = [record.getMessage() for record in caplog.records+ if record.name == type(provider).__module__ and record.levelno == logging.WARNING]+ assert len(warnings) == 1+ assert "recall timed out" in warnings[0]+ assert error.__name__ in warnings[0]+ assert "budget_s=1.5" in warnings[0]+ assert "request_s=0.75" in warnings[0]+ assert "private query" not in warnings[0]+ assert "private-identity" not in warnings[0]+ assert "private query and identity in exception" not in warnings[0]+ assert requests == ["/api/v1/search/find"]+++def test_recovered_session_search_timeout_is_quiet(recall_provider, caplog):+ provider, _module, requests, behavior = recall_provider+ behavior.update(error=httpx.ReadTimeout, fallback=True)+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):+ result = provider._search_prefetch_context(+ "what was the decision", session_id="recall-session", client=provider._client+ )+ assert "The decision is retained." in result+ assert requests == ["/api/v1/search/search", "/api/v1/search/find"]+ assert not [record for record in caplog.records if record.levelno >= logging.WARNING]+++def test_non_timeout_error_keeps_existing_debug_behavior(recall_provider, caplog):+ provider, _module, requests, behavior = recall_provider+ behavior["error"] = ValueError+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):+ assert provider._search_prefetch_context(+ "what was the decision", client=provider._client+ ) == ""+ assert requests == ["/api/v1/search/find"]+ assert not [record for record in caplog.records if record.levelno >= logging.WARNING]+++def configure_query_recall(provider, behavior, *, scope, compress):+ provider.save_config(+ {"recall_scope": scope, "recall_compress": compress}, str(behavior["home"])+ )+ provider.on_turn_start(1, "private query", author_id="private-sender")+ # A later turn has already received the once-per-session profile block.+ provider._profile_prefetched_sessions.add("recall-session")+++def assert_timeout_warning(caplog, provider, error):+ warnings = [record.getMessage() for record in caplog.records+ if record.name == type(provider).__module__ and record.levelno == logging.WARNING]+ assert len(warnings) == 1+ assert f"({error.__name__}; budget_s=1.5 request_s=0.75)" in warnings[0]+ assert "private" not in warnings[0]++[email protected]("compress", ["off", "server", "auto"])[email protected]("error", [httpx.ReadTimeout, TimeoutError])+def test_public_peer_identity_timeout_warns_and_stops(+ recall_provider, caplog, compress, error+):+ provider, _module, requests, behavior = recall_provider+ configure_query_recall(provider, behavior, scope="peer", compress=compress)+ behavior["error"] = error+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):+ assert provider.prefetch("private query", session_id="recall-session") == ""+ assert_timeout_warning(caplog, provider, error)+ assert requests == ["/api/v1/system/status"]++[email protected]("compress", ["off", "server", "auto"])[email protected]("scope", ["shared", "peer"])+def test_public_prefetch_exhausted_budget_warns_without_fallback_request(+ recall_provider, monkeypatch, caplog, compress, scope+):+ provider, module, requests, behavior = recall_provider+ configure_query_recall(provider, behavior, scope=scope, compress=compress)+ clock = SimpleNamespace(now=100.0)+ local_time = SimpleNamespace(**vars(module.time))+ local_time.monotonic = lambda: clock.now+ monkeypatch.setattr(module, "time", local_time)++ def respond(request):+ if request.url.path == "/api/v1/system/status":+ return httpx.Response(200, json={"result": {"user": "private-identity"}})+ assert request.url.path == "/api/v1/search/search"+ assert (json.loads(request.content).get("mode") == "context") == (compress != "off")+ # Consume the entire budget in the first search. The real deadline+ # check must prevent a second HTTP request and report TimeoutError.+ clock.now += 1.5+ raise httpx.ReadTimeout("private query and identity in exception")++ behavior["respond"] = respond+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):+ assert provider.prefetch("private query", session_id="recall-session") == ""+ assert_timeout_warning(caplog, provider, TimeoutError)+ assert requests == (+ ["/api/v1/system/status"] if scope == "peer" else []+ ) + ["/api/v1/search/search"]++[email protected]("error", [httpx.ReadTimeout, ValueError])+def test_identity_probe_default_preserves_existing_fallback(+ recall_provider, caplog, error+):+ provider, module, requests, behavior = recall_provider+ behavior["error"] = error+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):+ assert module._resolve_user_space(provider._client) is None+ assert requests == ["/api/v1/system/status"]+ assert not [record for record in caplog.records if record.levelno >= logging.WARNING]++[email protected]("compress", ["server", "auto"])+def test_public_compression_timeout_with_successful_fallback_is_quiet(+ recall_provider, caplog, compress+):+ provider, _module, requests, behavior = recall_provider+ configure_query_recall(provider, behavior, scope="shared", compress=compress)++ def respond(request):+ if json.loads(request.content).get("mode") == "context":+ raise httpx.ReadTimeout("private query and identity in exception")+ return httpx.Response(200, json={"result": {"memories": [{+ "uri": "viking://user/private-identity/memories/decision.md",+ "abstract": "The decision is retained.", "score": 0.9,+ }]}})++ behavior["respond"] = respond+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):+ result = provider.prefetch("private query", session_id="recall-session")+ assert "The decision is retained." in result+ assert requests == ["/api/v1/search/search", "/api/v1/search/search"]+ assert not [record for record in caplog.records if record.levelno >= logging.WARNING]