Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(hermes-plugin): classify underlying Quick Local LLM errors (#5643)

z1gonc5b5229a0edd
diff --git a/examples/hermes-plugin/quick_local.py b/examples/hermes-plugin/quick_local.py
index 962efb369..4fc6d1714 100644
--- a/examples/hermes-plugin/quick_local.py
+++ b/examples/hermes-plugin/quick_local.py
@@ -653,6 +653,25 @@ def _validate_vlm(paths: QuickLocalPaths, config_path: Path) -> None:
import json, sys, time
import httpx, openai
from openviking.models.vlm import VLMFactory
+
+def failure_details(exc):
+ # LiteLLM can replace an Anthropic HTTP/transport error with synthetic 500.
+ # Stop at the first HTTPX error: its context may contain an older failure.
+ error, current, seen = exc, exc, set()
+ while isinstance(current, BaseException) and id(current) not in seen:
+ seen.add(id(current))
+ if isinstance(current, (httpx.HTTPError, TimeoutError)):
+ error = current
+ break
+ # Suppression hides traceback text, not the structured HTTP metadata.
+ current = current.__cause__ or current.__context__
+ timed_out = isinstance(error, (TimeoutError, httpx.TimeoutException, openai.APITimeoutError))
+ connection = not timed_out and isinstance(error, (httpx.TransportError, openai.APIConnectionError))
+ status = getattr(getattr(error, "response", None), "status_code", getattr(error, "status_code", None))
+ status = status if type(status) is int and 100 <= status <= 599 and not (timed_out or connection) else None
+ return {"error": type(exc).__name__, "status": status,
+ "timeout": timed_out, "connection": connection}
+
for attempt in (1, 2):
try:
config = json.load(open(sys.argv[1], encoding="utf-8"))["vlm"]
@@ -662,15 +681,13 @@ for attempt in (1, 2):
raise ValueError("Empty completion")
break
except Exception as exc:
- status = getattr(exc, "status_code", None)
- status = status if type(status) is int and 100 <= status <= 599 else None
- timed_out = isinstance(exc, (TimeoutError, httpx.TimeoutException, openai.APITimeoutError))
- retry = status == 429 or (status is not None and 500 <= status <= 599) or timed_out
+ failure = failure_details(exc)
+ status = failure["status"]
+ retry = status == 429 or (status is not None and 500 <= status <= 599) or failure["timeout"]
if attempt == 1 and retry:
time.sleep(1)
continue
- print(json.dumps({"error": type(exc).__name__, "status": status,
- "timeout": timed_out, "attempts": attempt}))
+ print(json.dumps({**failure, "attempts": attempt}))
sys.exit(1)
"""
try:
@@ -722,7 +739,7 @@ def _vlm_check_failure(paths: QuickLocalPaths, failure: Any) -> QuickLocalSetupE
guidance = "The provider is rate-limiting requests. Wait, then retry setup."
elif status is not None and status >= 500:
guidance = "The provider reported a temporary server error. Retry setup later."
- elif name == "APIConnectionError":
+ elif failure.get("connection") is True or name == "APIConnectionError":
guidance = "Check the endpoint and network connection, then retry setup."
else:
guidance = "Check the model and endpoint. The model must support Chat Completions or Anthropic Messages."
diff --git a/examples/hermes-plugin/tests/test_quick_local_preflight.py b/examples/hermes-plugin/tests/test_quick_local_preflight.py
index 0543e06a4..d710cde4c 100644
--- a/examples/hermes-plugin/tests/test_quick_local_preflight.py
+++ b/examples/hermes-plugin/tests/test_quick_local_preflight.py
@@ -20,15 +20,25 @@ def ql(external_provider):
return importlib.import_module(module.__name__ + ".quick_local")
[email protected]("status,retries", [(401, False), (404, False), (429, True), (500, True)])
-def test_preflight_script_retries_only_transient_errors(ql, tmp_path, monkeypatch, status, retries):
[email protected]("status,name,retries", [
+ (401, "AuthenticationError", False),
+ (403, "PermissionDeniedError", False),
+ (404, "NotFoundError", False),
+ (429, "RateLimitError", True),
+ (500, "InternalServerError", True),
+])
+def test_preflight_script_retries_only_transient_errors(ql, tmp_path, monkeypatch, status, name, retries):
import openai
- error = openai.APIStatusError(
+ error = getattr(openai, name)(
"secret response body", response=httpx.Response(status, request=httpx.Request("POST", "https://test")),
body=None,
)
- _check_script(ql, tmp_path, monkeypatch, error, retries)
+ error.__context__ = httpx.HTTPStatusError(
+ "secret response body", request=error.request, response=error.response,
+ )
+ error.__suppress_context__ = True # The OpenAI SDK raises status errors from None.
+ _check_script(ql, tmp_path, monkeypatch, error, retries, succeeds=False)
@pytest.mark.parametrize("error,retries", [
@@ -55,7 +65,67 @@ def test_preflight_sdk_timeout_stops_after_one_retry(ql, tmp_path, monkeypatch):
succeeds=False)
-def _check_script(ql, tmp_path, monkeypatch, error, retries, succeeds=True):
[email protected]("status", [401, 403, 404, 429, 500, 503])
+def test_wrapped_http_status_controls_retry_and_guidance(ql, tmp_path, monkeypatch, status):
+ import openai
+
+ request = httpx.Request("POST", "https://test")
+ # LiteLLM can mask an Anthropic HTTP 403 as APIConnectionError(status=500).
+ error = openai.APIConnectionError(request=request)
+ error.status_code = 500
+ error.__context__ = httpx.HTTPStatusError(
+ "secret response body mentioning HTTP 500", request=request,
+ response=httpx.Response(status, request=request),
+ )
+ _check_script(ql, tmp_path, monkeypatch, error, status in {429, 500, 503},
+ succeeds=False, expected=f"HTTP {status}")
+
+
[email protected]("cause,retries,expected", [
+ (httpx.ConnectError("secret endpoint"), False, "endpoint and network"),
+ (httpx.ReadTimeout("secret endpoint"), True, "timed out"),
+ (TimeoutError("secret endpoint"), True, "timed out"),
+])
+def test_wrapped_transport_errors_ignore_synthetic_500(
+ ql, tmp_path, monkeypatch, cause, retries, expected,
+):
+ import openai
+
+ request = httpx.Request("POST", "https://test")
+ error = openai.InternalServerError(
+ "secret wrapper", response=httpx.Response(500, request=request), body=None,
+ )
+ error.__cause__ = cause
+ # Explicit causes take precedence over a different, implicit context.
+ error.__context__ = openai.AuthenticationError(
+ "secret earlier error", response=httpx.Response(401, request=request), body=None,
+ )
+ message = _check_script(ql, tmp_path, monkeypatch, error, retries,
+ succeeds=False, expected=expected)
+ assert "HTTP 500" not in message
+
+
+def test_cyclic_error_context_does_not_hang(ql, tmp_path, monkeypatch):
+ import openai
+
+ error = openai.APIConnectionError(request=httpx.Request("POST", "https://test"))
+ error.status_code = 500
+ error.__context__ = error
+ _check_script(ql, tmp_path, monkeypatch, error, False, expected="endpoint and network")
+
+
+def test_current_http_response_wins_over_earlier_context(ql, tmp_path, monkeypatch):
+ request = httpx.Request("POST", "https://test")
+ error = httpx.HTTPStatusError(
+ "secret current response", request=request, response=httpx.Response(503, request=request),
+ )
+ error.__context__ = httpx.HTTPStatusError(
+ "secret earlier response", request=request, response=httpx.Response(401, request=request),
+ )
+ _check_script(ql, tmp_path, monkeypatch, error, True, succeeds=False, expected="HTTP 503")
+
+
+def _check_script(ql, tmp_path, monkeypatch, error, retries, succeeds=True, expected=None):
"""Execute the shipped child script, replacing only the LLM backend."""
paths = ql.managed_paths(tmp_path)
config = tmp_path / "check.json"
@@ -84,17 +154,25 @@ def _check_script(ql, tmp_path, monkeypatch, error, retries, succeeds=True):
return subprocess.CompletedProcess(command, code, stdout=output.getvalue(), stderr="secret stderr")
monkeypatch.setattr(ql.subprocess, "run", run)
+ message = ""
if retries and succeeds:
ql._validate_vlm(paths, config)
else:
with pytest.raises(ql.QuickLocalSetupError) as caught:
ql._validate_vlm(paths, config)
assert "secret" not in str(caught.value)
+ message = str(caught.value)
+ log = (tmp_path / "logs/openviking-server.log").read_text()
+ assert "secret" not in log
+ assert type(error).__name__ in message and type(error).__name__ in log
+ if expected:
+ assert expected in message
assert vlm.get_completion.call_count == (2 if retries else 1)
if retries:
sleep.assert_called_once_with(1)
else:
sleep.assert_not_called()
+ return message
@pytest.mark.parametrize("failure,expected", [
diff --git a/scripts/check-hermes-quick-local.py b/scripts/check-hermes-quick-local.py
index 4dc2ed321..9b51f7efd 100644
--- a/scripts/check-hermes-quick-local.py
+++ b/scripts/check-hermes-quick-local.py
@@ -148,19 +148,26 @@ def main():
def do_POST(self):
self.rfile.read(int(self.headers["Content-Length"]))
- if self.path != "/v1/chat/completions" or self.headers.get("Authorization") not in {
- "Bearer isolated-static-test-key", "Bearer updated-isolated-static-key"
- }:
+ anthropic = self.path == "/anthropic/v1/messages"
+ key = (self.headers.get("x-api-key") if anthropic
+ else self.headers.get("Authorization", "").removeprefix("Bearer "))
+ if (self.path not in {"/v1/chat/completions", "/anthropic/v1/messages"}
+ or key not in {"isolated-static-test-key", "updated-isolated-static-key"}):
self.send_error(401)
return
llm_requests.append(self.path)
if preflight_errors:
self.send_error(preflight_errors.pop(0), "private provider response")
return
- body = json.dumps({"id": "setup-test", "object": "chat.completion", "model": "test-model",
- "choices": [{"index": 0, "message": {"role": "assistant", "content": "OK"},
- "finish_reason": "stop"}],
- "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}).encode()
+ body = json.dumps(
+ {"id": "msg-test", "type": "message", "role": "assistant", "model": "claude-test",
+ "content": [{"type": "text", "text": "OK"}], "stop_reason": "end_turn",
+ "usage": {"input_tokens": 1, "output_tokens": 1}} if anthropic else
+ {"id": "setup-test", "object": "chat.completion", "model": "test-model",
+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "OK"},
+ "finish_reason": "stop"}],
+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}
+ ).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
@@ -234,23 +241,31 @@ def main():
root / "transport-config", ql, server.paths.runtime_python
)
evidence["llm_preflight"] = []
- for errors in ([401], [404], [429], [500], [429, 429]):
- preflight_errors[:] = errors
- count = len(llm_requests)
- log_path = home / "logs/openviking-server.log"
- log_size = log_path.stat().st_size
- failed = False
- try:
- ql._validate_vlm(server.paths, server.paths.server_config)
- except ql.QuickLocalSetupError as error:
- failed = True
- assert f"HTTP {errors[-1]}" in str(error), str(error)
- assert "private provider response" not in str(error)
- assert failed == (errors[0] in {401, 404} or len(errors) == 2)
- expected = 1 if errors[0] in {401, 404} else 2
- assert len(llm_requests) - count == expected
- assert "private provider response" not in log_path.read_bytes()[log_size:].decode()
- evidence["llm_preflight"].append({"errors": errors, "requests": expected, "failed": failed})
+ for route in ("openai", "anthropic"):
+ check_config = json.loads(server.paths.server_config.read_text())
+ if route == "anthropic":
+ check_config["vlm"].update(provider="litellm", model="anthropic/claude-test",
+ api_base=f"http://127.0.0.1:{llm_server.server_port}/anthropic")
+ check_path = root / "preflight-check.json"
+ check_path.write_text(json.dumps(check_config), encoding="utf-8")
+ for errors in ([401], [403], [404], [429], [500], [503], [429, 429]):
+ preflight_errors[:] = errors
+ count = len(llm_requests)
+ log_path = home / "logs/openviking-server.log"
+ log_size = log_path.stat().st_size
+ failed = False
+ try:
+ ql._validate_vlm(server.paths, check_path)
+ except ql.QuickLocalSetupError as error:
+ failed = True
+ assert f"HTTP {errors[-1]}" in str(error), str(error)
+ assert "private provider response" not in str(error)
+ assert failed == (errors[0] in {401, 403, 404} or len(errors) == 2)
+ expected = 1 if errors[0] in {401, 403, 404} else 2
+ assert len(llm_requests) - count == expected
+ assert "private provider response" not in log_path.read_bytes()[log_size:].decode()
+ evidence["llm_preflight"].append({"route": route, "errors": errors,
+ "requests": expected, "failed": failed})
assert not preflight_errors
provider.initialize("local-smoke-" + name, hermes_home=str(home), platform="cli")
assert provider._client is not None