Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(hermes-plugin): classify underlying Quick Local LLM errors (#5643)
z1gonc5b5229a0edd
diff --git a/examples/hermes-plugin/quick_local.py b/examples/hermes-plugin/quick_local.pyindex 962efb369..4fc6d1714 100644--- a/examples/hermes-plugin/quick_local.py+++ b/examples/hermes-plugin/quick_local.py@@ -653,6 +653,25 @@ def _validate_vlm(paths: QuickLocalPaths, config_path: Path) -> None:import json, sys, timeimport httpx, openaifrom openviking.models.vlm import VLMFactory++def failure_details(exc):+ # LiteLLM can replace an Anthropic HTTP/transport error with synthetic 500.+ # Stop at the first HTTPX error: its context may contain an older failure.+ error, current, seen = exc, exc, set()+ while isinstance(current, BaseException) and id(current) not in seen:+ seen.add(id(current))+ if isinstance(current, (httpx.HTTPError, TimeoutError)):+ error = current+ break+ # Suppression hides traceback text, not the structured HTTP metadata.+ current = current.__cause__ or current.__context__+ timed_out = isinstance(error, (TimeoutError, httpx.TimeoutException, openai.APITimeoutError))+ connection = not timed_out and isinstance(error, (httpx.TransportError, openai.APIConnectionError))+ status = getattr(getattr(error, "response", None), "status_code", getattr(error, "status_code", None))+ status = status if type(status) is int and 100 <= status <= 599 and not (timed_out or connection) else None+ return {"error": type(exc).__name__, "status": status,+ "timeout": timed_out, "connection": connection}+for attempt in (1, 2):try:config = json.load(open(sys.argv[1], encoding="utf-8"))["vlm"]@@ -662,15 +681,13 @@ for attempt in (1, 2):raise ValueError("Empty completion")breakexcept Exception as exc:- status = getattr(exc, "status_code", None)- status = status if type(status) is int and 100 <= status <= 599 else None- timed_out = isinstance(exc, (TimeoutError, httpx.TimeoutException, openai.APITimeoutError))- retry = status == 429 or (status is not None and 500 <= status <= 599) or timed_out+ failure = failure_details(exc)+ status = failure["status"]+ retry = status == 429 or (status is not None and 500 <= status <= 599) or failure["timeout"]if attempt == 1 and retry:time.sleep(1)continue- print(json.dumps({"error": type(exc).__name__, "status": status,- "timeout": timed_out, "attempts": attempt}))+ print(json.dumps({**failure, "attempts": attempt}))sys.exit(1)"""try:@@ -722,7 +739,7 @@ def _vlm_check_failure(paths: QuickLocalPaths, failure: Any) -> QuickLocalSetupEguidance = "The provider is rate-limiting requests. Wait, then retry setup."elif status is not None and status >= 500:guidance = "The provider reported a temporary server error. Retry setup later."- elif name == "APIConnectionError":+ elif failure.get("connection") is True or name == "APIConnectionError":guidance = "Check the endpoint and network connection, then retry setup."else:guidance = "Check the model and endpoint. The model must support Chat Completions or Anthropic Messages."diff --git a/examples/hermes-plugin/tests/test_quick_local_preflight.py b/examples/hermes-plugin/tests/test_quick_local_preflight.pyindex 0543e06a4..d710cde4c 100644--- a/examples/hermes-plugin/tests/test_quick_local_preflight.py+++ b/examples/hermes-plugin/tests/test_quick_local_preflight.py@@ -20,15 +20,25 @@ def ql(external_provider):return importlib.import_module(module.__name__ + ".quick_local")[email protected]("status,retries", [(401, False), (404, False), (429, True), (500, True)])-def test_preflight_script_retries_only_transient_errors(ql, tmp_path, monkeypatch, status, retries):[email protected]("status,name,retries", [+ (401, "AuthenticationError", False),+ (403, "PermissionDeniedError", False),+ (404, "NotFoundError", False),+ (429, "RateLimitError", True),+ (500, "InternalServerError", True),+])+def test_preflight_script_retries_only_transient_errors(ql, tmp_path, monkeypatch, status, name, retries):import openai- error = openai.APIStatusError(+ error = getattr(openai, name)("secret response body", response=httpx.Response(status, request=httpx.Request("POST", "https://test")),body=None,)- _check_script(ql, tmp_path, monkeypatch, error, retries)+ error.__context__ = httpx.HTTPStatusError(+ "secret response body", request=error.request, response=error.response,+ )+ error.__suppress_context__ = True # The OpenAI SDK raises status errors from None.+ _check_script(ql, tmp_path, monkeypatch, error, retries, succeeds=False)@pytest.mark.parametrize("error,retries", [@@ -55,7 +65,67 @@ def test_preflight_sdk_timeout_stops_after_one_retry(ql, tmp_path, monkeypatch):succeeds=False)-def _check_script(ql, tmp_path, monkeypatch, error, retries, succeeds=True):[email protected]("status", [401, 403, 404, 429, 500, 503])+def test_wrapped_http_status_controls_retry_and_guidance(ql, tmp_path, monkeypatch, status):+ import openai++ request = httpx.Request("POST", "https://test")+ # LiteLLM can mask an Anthropic HTTP 403 as APIConnectionError(status=500).+ error = openai.APIConnectionError(request=request)+ error.status_code = 500+ error.__context__ = httpx.HTTPStatusError(+ "secret response body mentioning HTTP 500", request=request,+ response=httpx.Response(status, request=request),+ )+ _check_script(ql, tmp_path, monkeypatch, error, status in {429, 500, 503},+ succeeds=False, expected=f"HTTP {status}")++[email protected]("cause,retries,expected", [+ (httpx.ConnectError("secret endpoint"), False, "endpoint and network"),+ (httpx.ReadTimeout("secret endpoint"), True, "timed out"),+ (TimeoutError("secret endpoint"), True, "timed out"),+])+def test_wrapped_transport_errors_ignore_synthetic_500(+ ql, tmp_path, monkeypatch, cause, retries, expected,+):+ import openai++ request = httpx.Request("POST", "https://test")+ error = openai.InternalServerError(+ "secret wrapper", response=httpx.Response(500, request=request), body=None,+ )+ error.__cause__ = cause+ # Explicit causes take precedence over a different, implicit context.+ error.__context__ = openai.AuthenticationError(+ "secret earlier error", response=httpx.Response(401, request=request), body=None,+ )+ message = _check_script(ql, tmp_path, monkeypatch, error, retries,+ succeeds=False, expected=expected)+ assert "HTTP 500" not in message+++def test_cyclic_error_context_does_not_hang(ql, tmp_path, monkeypatch):+ import openai++ error = openai.APIConnectionError(request=httpx.Request("POST", "https://test"))+ error.status_code = 500+ error.__context__ = error+ _check_script(ql, tmp_path, monkeypatch, error, False, expected="endpoint and network")+++def test_current_http_response_wins_over_earlier_context(ql, tmp_path, monkeypatch):+ request = httpx.Request("POST", "https://test")+ error = httpx.HTTPStatusError(+ "secret current response", request=request, response=httpx.Response(503, request=request),+ )+ error.__context__ = httpx.HTTPStatusError(+ "secret earlier response", request=request, response=httpx.Response(401, request=request),+ )+ _check_script(ql, tmp_path, monkeypatch, error, True, succeeds=False, expected="HTTP 503")+++def _check_script(ql, tmp_path, monkeypatch, error, retries, succeeds=True, expected=None):"""Execute the shipped child script, replacing only the LLM backend."""paths = ql.managed_paths(tmp_path)config = tmp_path / "check.json"@@ -84,17 +154,25 @@ def _check_script(ql, tmp_path, monkeypatch, error, retries, succeeds=True):return subprocess.CompletedProcess(command, code, stdout=output.getvalue(), stderr="secret stderr")monkeypatch.setattr(ql.subprocess, "run", run)+ message = ""if retries and succeeds:ql._validate_vlm(paths, config)else:with pytest.raises(ql.QuickLocalSetupError) as caught:ql._validate_vlm(paths, config)assert "secret" not in str(caught.value)+ message = str(caught.value)+ log = (tmp_path / "logs/openviking-server.log").read_text()+ assert "secret" not in log+ assert type(error).__name__ in message and type(error).__name__ in log+ if expected:+ assert expected in messageassert vlm.get_completion.call_count == (2 if retries else 1)if retries:sleep.assert_called_once_with(1)else:sleep.assert_not_called()+ return message@pytest.mark.parametrize("failure,expected", [diff --git a/scripts/check-hermes-quick-local.py b/scripts/check-hermes-quick-local.pyindex 4dc2ed321..9b51f7efd 100644--- a/scripts/check-hermes-quick-local.py+++ b/scripts/check-hermes-quick-local.py@@ -148,19 +148,26 @@ def main():def do_POST(self):self.rfile.read(int(self.headers["Content-Length"]))- if self.path != "/v1/chat/completions" or self.headers.get("Authorization") not in {- "Bearer isolated-static-test-key", "Bearer updated-isolated-static-key"- }:+ anthropic = self.path == "/anthropic/v1/messages"+ key = (self.headers.get("x-api-key") if anthropic+ else self.headers.get("Authorization", "").removeprefix("Bearer "))+ if (self.path not in {"/v1/chat/completions", "/anthropic/v1/messages"}+ or key not in {"isolated-static-test-key", "updated-isolated-static-key"}):self.send_error(401)returnllm_requests.append(self.path)if preflight_errors:self.send_error(preflight_errors.pop(0), "private provider response")return- body = json.dumps({"id": "setup-test", "object": "chat.completion", "model": "test-model",- "choices": [{"index": 0, "message": {"role": "assistant", "content": "OK"},- "finish_reason": "stop"}],- "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}).encode()+ body = json.dumps(+ {"id": "msg-test", "type": "message", "role": "assistant", "model": "claude-test",+ "content": [{"type": "text", "text": "OK"}], "stop_reason": "end_turn",+ "usage": {"input_tokens": 1, "output_tokens": 1}} if anthropic else+ {"id": "setup-test", "object": "chat.completion", "model": "test-model",+ "choices": [{"index": 0, "message": {"role": "assistant", "content": "OK"},+ "finish_reason": "stop"}],+ "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}+ ).encode()self.send_response(200)self.send_header("Content-Type", "application/json")self.send_header("Content-Length", str(len(body)))@@ -234,23 +241,31 @@ def main():root / "transport-config", ql, server.paths.runtime_python)evidence["llm_preflight"] = []- for errors in ([401], [404], [429], [500], [429, 429]):- preflight_errors[:] = errors- count = len(llm_requests)- log_path = home / "logs/openviking-server.log"- log_size = log_path.stat().st_size- failed = False- try:- ql._validate_vlm(server.paths, server.paths.server_config)- except ql.QuickLocalSetupError as error:- failed = True- assert f"HTTP {errors[-1]}" in str(error), str(error)- assert "private provider response" not in str(error)- assert failed == (errors[0] in {401, 404} or len(errors) == 2)- expected = 1 if errors[0] in {401, 404} else 2- assert len(llm_requests) - count == expected- assert "private provider response" not in log_path.read_bytes()[log_size:].decode()- evidence["llm_preflight"].append({"errors": errors, "requests": expected, "failed": failed})+ for route in ("openai", "anthropic"):+ check_config = json.loads(server.paths.server_config.read_text())+ if route == "anthropic":+ check_config["vlm"].update(provider="litellm", model="anthropic/claude-test",+ api_base=f"http://127.0.0.1:{llm_server.server_port}/anthropic")+ check_path = root / "preflight-check.json"+ check_path.write_text(json.dumps(check_config), encoding="utf-8")+ for errors in ([401], [403], [404], [429], [500], [503], [429, 429]):+ preflight_errors[:] = errors+ count = len(llm_requests)+ log_path = home / "logs/openviking-server.log"+ log_size = log_path.stat().st_size+ failed = False+ try:+ ql._validate_vlm(server.paths, check_path)+ except ql.QuickLocalSetupError as error:+ failed = True+ assert f"HTTP {errors[-1]}" in str(error), str(error)+ assert "private provider response" not in str(error)+ assert failed == (errors[0] in {401, 403, 404} or len(errors) == 2)+ expected = 1 if errors[0] in {401, 403, 404} else 2+ assert len(llm_requests) - count == expected+ assert "private provider response" not in log_path.read_bytes()[log_size:].decode()+ evidence["llm_preflight"].append({"route": route, "errors": errors,+ "requests": expected, "failed": failed})assert not preflight_errorsprovider.initialize("local-smoke-" + name, hermes_home=str(home), platform="cli")assert provider._client is not None