Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(hermes-plugin): report query recall timeouts (#5391)

z1gonec646203e960
diff --git a/examples/hermes-plugin/__init__.py b/examples/hermes-plugin/__init__.py
index b2bdfdc9f..488d7c4bb 100644
--- a/examples/hermes-plugin/__init__.py
+++ b/examples/hermes-plugin/__init__.py
@@ -268,6 +268,17 @@ def _get_httpx():
return None
+def _is_timeout_error(error: BaseException) -> bool:
+ """Identify socket and HTTP transport timeouts in query recall."""
+ if isinstance(error, TimeoutError):
+ return True
+ try:
+ from httpx import TimeoutException
+ except ImportError:
+ return False
+ return isinstance(error, TimeoutException)
+
+
class _VikingClient:
"""Thin HTTP client for the OpenViking REST API (httpx, no SDK dependency)."""
@@ -500,13 +511,17 @@ _TOOL_STATUS_ERROR_ALIASES = {"error", "failed", "failure"}
_TOOL_STATUS_COMPLETED_ALIASES = {"completed", "complete", "success", "succeeded"}
-def _resolve_user_space(client, *, timeout: Optional[float] = None) -> Optional[str]:
+def _resolve_user_space(client, *, timeout: Optional[float] = None,
+ raise_on_timeout: bool = False) -> Optional[str]:
"""Server-asserted current user for explicit-uid URIs; ``None`` when the probe fails or
reports no user. Callers may fall back to a configured value for that one operation but
- must not cache an unverified identity — a later probe can succeed."""
+ must not cache an unverified identity — a later probe can succeed.
+ Query recall can propagate timeouts to its bounded warning handler."""
try:
status = client.get("/api/v1/system/status", **({"timeout": timeout} if timeout is not None else {}))
- except Exception:
+ except Exception as exc:
+ if raise_on_timeout and _is_timeout_error(exc):
+ raise
logger.debug("OpenViking user-space probe failed; using configured fallback", exc_info=True)
return None
return str(((status or {}).get("result") or {}).get("user") or "").strip() or None
@@ -1808,6 +1823,7 @@ class OpenVikingMemoryProvider(MemoryProvider):
if client is None:
return ""
+ cfg = None
try:
cfg = self._recall_config()
deadline = time.monotonic() + cfg["timeout_seconds"]
@@ -1820,7 +1836,10 @@ class OpenVikingMemoryProvider(MemoryProvider):
if scope == "peer":
# Explicit roots also constrain fallback searches when there is
# no sender. An actor-less user-root search includes all peers.
- user = _resolve_user_space(client, timeout=self._remaining_recall_timeout(deadline, cfg["request_timeout_seconds"]))
+ user = _resolve_user_space(
+ client, timeout=self._remaining_recall_timeout(deadline, cfg["request_timeout_seconds"]),
+ raise_on_timeout=True,
+ )
if not user:
return ""
user_root = f"viking://user/{user}"
@@ -1896,7 +1915,15 @@ class OpenVikingMemoryProvider(MemoryProvider):
deadline=deadline, request_timeout=cfg["request_timeout_seconds"], full_read_limit=cfg["full_read_limit"],
))
except Exception as e:
- logger.debug("OpenViking context search failed: %s", e)
+ # A timeout leaves query recall empty. Report only local, bounded
+ # diagnostics; exception text can contain query or identity data.
+ if cfg is not None and _is_timeout_error(e):
+ logger.warning(
+ "OpenViking recall timed out (%s; budget_s=%s request_s=%s); no query context injected",
+ type(e).__name__, cfg["timeout_seconds"], cfg["request_timeout_seconds"],
+ )
+ else:
+ logger.debug("OpenViking context search failed: %s", e)
return ""
# -- typed settings ------------------------------------------------------
diff --git a/examples/hermes-plugin/tests/test_recall_timeout.py b/examples/hermes-plugin/tests/test_recall_timeout.py
new file mode 100644
index 000000000..95a8e9439
--- /dev/null
+++ b/examples/hermes-plugin/tests/test_recall_timeout.py
@@ -0,0 +1,195 @@
+"""Recall timeout diagnostics through the installed external provider."""
+
+import json
+import logging
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+
+def recall_provider(external_provider, monkeypatch):
+ home, provider, module, _ = external_provider("recall-timeout")
+ (home / "config.yaml").write_text(
+ "memory:\n provider: openviking\n openviking:\n"
+ " endpoint: http://openviking.test\n"
+ " use_ovcli_config: false\n"
+ " recall_timeout_seconds: 1.5\n"
+ " recall_request_timeout_seconds: 0.75\n"
+ " recall_prefer_abstract: true\n",
+ encoding="utf-8",
+ )
+ requests = []
+ behavior = {"error": None, "fallback": False, "respond": None, "home": home}
+
+ def handle(request):
+ requests.append(request.url.path)
+ if behavior["respond"] is not None:
+ return behavior["respond"](request)
+ if behavior["error"] is not None and not (
+ behavior["fallback"] and request.url.path == "/api/v1/search/find"
+ ):
+ raise behavior["error"]("private query and identity in exception")
+ return httpx.Response(
+ 200,
+ json={"result": {"memories": [{
+ "uri": "viking://user/private-identity/memories/decision.md",
+ "abstract": "The decision is retained.",
+ "score": 0.9,
+ }]}},
+ )
+
+ with httpx.Client(transport=httpx.MockTransport(handle)) as client:
+ monkeypatch.setattr(httpx, "get", client.get)
+ monkeypatch.setattr(httpx, "post", client.post)
+ monkeypatch.setattr(module, "_classify_runtime_openviking_health", lambda *_: ("healthy", ""))
+ provider.initialize(
+ "recall-session", hermes_home=str(home), platform="telegram", user_id="private-sender"
+ )
+ yield provider, module, requests, behavior
+
+
+ httpx.ReadTimeout, httpx.ConnectTimeout, httpx.WriteTimeout,
+ httpx.PoolTimeout, TimeoutError,
+])
+def test_timeout_warns_without_private_text_or_extra_requests(recall_provider, caplog, error):
+ provider, _module, requests, behavior = recall_provider
+ behavior["error"] = error
+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):
+ assert provider._search_prefetch_context(
+ "private query", client=provider._client
+ ) == ""
+
+ warnings = [record.getMessage() for record in caplog.records
+ if record.name == type(provider).__module__ and record.levelno == logging.WARNING]
+ assert len(warnings) == 1
+ assert "recall timed out" in warnings[0]
+ assert error.__name__ in warnings[0]
+ assert "budget_s=1.5" in warnings[0]
+ assert "request_s=0.75" in warnings[0]
+ assert "private query" not in warnings[0]
+ assert "private-identity" not in warnings[0]
+ assert "private query and identity in exception" not in warnings[0]
+ assert requests == ["/api/v1/search/find"]
+
+
+def test_recovered_session_search_timeout_is_quiet(recall_provider, caplog):
+ provider, _module, requests, behavior = recall_provider
+ behavior.update(error=httpx.ReadTimeout, fallback=True)
+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):
+ result = provider._search_prefetch_context(
+ "what was the decision", session_id="recall-session", client=provider._client
+ )
+ assert "The decision is retained." in result
+ assert requests == ["/api/v1/search/search", "/api/v1/search/find"]
+ assert not [record for record in caplog.records if record.levelno >= logging.WARNING]
+
+
+def test_non_timeout_error_keeps_existing_debug_behavior(recall_provider, caplog):
+ provider, _module, requests, behavior = recall_provider
+ behavior["error"] = ValueError
+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):
+ assert provider._search_prefetch_context(
+ "what was the decision", client=provider._client
+ ) == ""
+ assert requests == ["/api/v1/search/find"]
+ assert not [record for record in caplog.records if record.levelno >= logging.WARNING]
+
+
+def configure_query_recall(provider, behavior, *, scope, compress):
+ provider.save_config(
+ {"recall_scope": scope, "recall_compress": compress}, str(behavior["home"])
+ )
+ provider.on_turn_start(1, "private query", author_id="private-sender")
+ # A later turn has already received the once-per-session profile block.
+ provider._profile_prefetched_sessions.add("recall-session")
+
+
+def assert_timeout_warning(caplog, provider, error):
+ warnings = [record.getMessage() for record in caplog.records
+ if record.name == type(provider).__module__ and record.levelno == logging.WARNING]
+ assert len(warnings) == 1
+ assert f"({error.__name__}; budget_s=1.5 request_s=0.75)" in warnings[0]
+ assert "private" not in warnings[0]
+
+
[email protected]("compress", ["off", "server", "auto"])
[email protected]("error", [httpx.ReadTimeout, TimeoutError])
+def test_public_peer_identity_timeout_warns_and_stops(
+ recall_provider, caplog, compress, error
+):
+ provider, _module, requests, behavior = recall_provider
+ configure_query_recall(provider, behavior, scope="peer", compress=compress)
+ behavior["error"] = error
+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):
+ assert provider.prefetch("private query", session_id="recall-session") == ""
+ assert_timeout_warning(caplog, provider, error)
+ assert requests == ["/api/v1/system/status"]
+
+
[email protected]("compress", ["off", "server", "auto"])
[email protected]("scope", ["shared", "peer"])
+def test_public_prefetch_exhausted_budget_warns_without_fallback_request(
+ recall_provider, monkeypatch, caplog, compress, scope
+):
+ provider, module, requests, behavior = recall_provider
+ configure_query_recall(provider, behavior, scope=scope, compress=compress)
+ clock = SimpleNamespace(now=100.0)
+ local_time = SimpleNamespace(**vars(module.time))
+ local_time.monotonic = lambda: clock.now
+ monkeypatch.setattr(module, "time", local_time)
+
+ def respond(request):
+ if request.url.path == "/api/v1/system/status":
+ return httpx.Response(200, json={"result": {"user": "private-identity"}})
+ assert request.url.path == "/api/v1/search/search"
+ assert (json.loads(request.content).get("mode") == "context") == (compress != "off")
+ # Consume the entire budget in the first search. The real deadline
+ # check must prevent a second HTTP request and report TimeoutError.
+ clock.now += 1.5
+ raise httpx.ReadTimeout("private query and identity in exception")
+
+ behavior["respond"] = respond
+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):
+ assert provider.prefetch("private query", session_id="recall-session") == ""
+ assert_timeout_warning(caplog, provider, TimeoutError)
+ assert requests == (
+ ["/api/v1/system/status"] if scope == "peer" else []
+ ) + ["/api/v1/search/search"]
+
+
[email protected]("error", [httpx.ReadTimeout, ValueError])
+def test_identity_probe_default_preserves_existing_fallback(
+ recall_provider, caplog, error
+):
+ provider, module, requests, behavior = recall_provider
+ behavior["error"] = error
+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):
+ assert module._resolve_user_space(provider._client) is None
+ assert requests == ["/api/v1/system/status"]
+ assert not [record for record in caplog.records if record.levelno >= logging.WARNING]
+
+
[email protected]("compress", ["server", "auto"])
+def test_public_compression_timeout_with_successful_fallback_is_quiet(
+ recall_provider, caplog, compress
+):
+ provider, _module, requests, behavior = recall_provider
+ configure_query_recall(provider, behavior, scope="shared", compress=compress)
+
+ def respond(request):
+ if json.loads(request.content).get("mode") == "context":
+ raise httpx.ReadTimeout("private query and identity in exception")
+ return httpx.Response(200, json={"result": {"memories": [{
+ "uri": "viking://user/private-identity/memories/decision.md",
+ "abstract": "The decision is retained.", "score": 0.9,
+ }]}})
+
+ behavior["respond"] = respond
+ with caplog.at_level(logging.WARNING, logger=type(provider).__module__):
+ result = provider.prefetch("private query", session_id="recall-session")
+ assert "The decision is retained." in result
+ assert requests == ["/api/v1/search/search", "/api/v1/search/search"]
+ assert not [record for record in caplog.records if record.levelno >= logging.WARNING]