Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: improve Ark VLM error diagnostics (#4072)
DuTaof3202001f731
diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.pyindex aaa8a09fa..9787bf491 100644--- a/bot/vikingbot/providers/vlm_adapter.py+++ b/bot/vikingbot/providers/vlm_adapter.py@@ -6,12 +6,16 @@ configuration semantics are consistent with openviking server's vlm section."""import asyncio+import json+import reimport time-from collections.abc import AsyncIterator+import traceback+from collections.abc import AsyncIterator, Mappingfrom typing import Anyfrom loguru import logger+from openviking.models.vlm.backends.volcengine_vlm import build_volcengine_request_headersfrom openviking.utils.model_retry import is_retryable_rate_limit_error, rate_limit_retry_delayfrom openviking.utils.multimodal import redact_image_data_urlsfrom vikingbot.integrations.langfuse import LangfuseClient@@ -26,6 +30,203 @@ from vikingbot.providers.base import ()from vikingbot.utils.tracing import get_current_response_id+_MAX_ERROR_LOG_VALUE_CHARS = 4096+_MAX_ERROR_LOG_ITEMS = 50+_SENSITIVE_ERROR_KEYS = frozenset(+ {+ "access_token",+ "api_key",+ "apikey",+ "authorization",+ "client_secret",+ "credential",+ "credentials",+ "id_token",+ "password",+ "refresh_token",+ "secret",+ "session_token",+ "token",+ }+)+_SENSITIVE_ERROR_KEY_SUFFIXES = (+ "api_key",+ "password",+ "secret",+ "token",+)+_SENSITIVE_ERROR_KV_RE = re.compile(+ r"(?i)((?:api[_-]?key|authorization|credential|password|secret|token)"+ r"[\"']?\s*[:=]\s*[\"']?)[^\s,;\]\}\"']+"+)+_BEARER_TOKEN_RE = re.compile(r"(?i)(bearer\s+)[^\s,;\]\}\"']+")+_API_KEY_TOKEN_RE = re.compile(r"\bsk-[a-zA-Z0-9._-]{8,}")+_SERVER_REQUEST_ID_HEADERS = ("x-request-id",)+_SERVER_TRACE_ID_HEADERS = (+ "x-trace-id",+ "trace-id",+ "x-tt-logid",+ "x-log-id",+)+++def _safe_exception_attribute(error: BaseException, name: str) -> Any:+ try:+ return getattr(error, name, None)+ except BaseException:+ return None+++def _exception_chain(error: BaseException) -> list[BaseException]:+ chain: list[BaseException] = []+ seen: set[int] = set()+ current: BaseException | None = error+ while current is not None and id(current) not in seen:+ seen.add(id(current))+ chain.append(current)+ current = _safe_exception_attribute(current, "__cause__") or _safe_exception_attribute(+ current, "__context__"+ )+ return chain+++def _redact_error_text(value: str) -> str:+ value = redact_image_data_urls(value)+ value = _SENSITIVE_ERROR_KV_RE.sub(lambda match: f"{match.group(1)}<redacted>", value)+ value = _BEARER_TOKEN_RE.sub(lambda match: f"{match.group(1)}<redacted>", value)+ value = _API_KEY_TOKEN_RE.sub("<redacted>", value)+ if len(value) > _MAX_ERROR_LOG_VALUE_CHARS:+ return value[:_MAX_ERROR_LOG_VALUE_CHARS] + "...[truncated]"+ return value+++def _sanitize_error_value(value: Any, *, depth: int = 0) -> Any:+ """Prepare an upstream error body for logs without exposing credentials or huge values."""+ if depth >= 5:+ return "<max-depth>"+ if isinstance(value, Mapping):+ sanitized: dict[str, Any] = {}+ for index, (key, item) in enumerate(value.items()):+ if index >= _MAX_ERROR_LOG_ITEMS:+ sanitized["<truncated>"] = f"{len(value) - _MAX_ERROR_LOG_ITEMS} more items"+ break+ key_text = str(key)+ normalized_key = key_text.lower().replace("-", "_")+ if normalized_key in _SENSITIVE_ERROR_KEYS or any(+ normalized_key.endswith(f"_{suffix}") for suffix in _SENSITIVE_ERROR_KEY_SUFFIXES+ ):+ sanitized[key_text] = "<redacted>"+ else:+ sanitized[key_text] = _sanitize_error_value(item, depth=depth + 1)+ return sanitized+ if isinstance(value, (list, tuple, set, frozenset)):+ items = list(value)+ sanitized_items = [+ _sanitize_error_value(item, depth=depth + 1) for item in items[:_MAX_ERROR_LOG_ITEMS]+ ]+ if len(items) > _MAX_ERROR_LOG_ITEMS:+ sanitized_items.append(f"<{len(items) - _MAX_ERROR_LOG_ITEMS} more items>")+ return sanitized_items+ if isinstance(value, bytes):+ value = value.decode("utf-8", errors="replace")+ if isinstance(value, str):+ return _redact_error_text(value)+ if value is None or isinstance(value, (bool, int, float)):+ return value+ try:+ return _redact_error_text(str(value))+ except BaseException:+ return f"<{type(value).__name__}>"+++def _render_error_log_value(value: Any) -> str:+ if value is None:+ return "-"+ sanitized = _sanitize_error_value(value)+ try:+ rendered = json.dumps(sanitized, ensure_ascii=False, default=str, separators=(",", ":"))+ except BaseException:+ rendered = _redact_error_text(str(sanitized))+ if len(rendered) > _MAX_ERROR_LOG_VALUE_CHARS:+ return rendered[:_MAX_ERROR_LOG_VALUE_CHARS] + "...[truncated]"+ return rendered+++def _first_exception_attribute(chain: list[BaseException], name: str) -> Any:+ for item in chain:+ value = _safe_exception_attribute(item, name)+ if value is not None:+ return value+ return None+++def _first_header_value(headers: Any, names: tuple[str, ...]) -> str | None:+ if headers is None:+ return None+ for name in names:+ try:+ value = headers.get(name)+ except BaseException:+ value = None+ if value:+ return _redact_error_text(str(value))+ return None+++def _exception_log_details(error: BaseException) -> dict[str, str]:+ """Extract Ark/OpenAI-compatible exception details for one structured log line."""+ chain = _exception_chain(error)+ response = _first_exception_attribute(chain, "response")+ request = _first_exception_attribute(chain, "request")+ if request is None and response is not None:+ request = _safe_exception_attribute(response, "request")++ status = _first_exception_attribute(chain, "status_code")+ if status is None and response is not None:+ status = _safe_exception_attribute(response, "status_code")++ body = _first_exception_attribute(chain, "body")+ if body is None and response is not None:+ body = _safe_exception_attribute(response, "text")++ response_headers = (+ _safe_exception_attribute(response, "headers") if response is not None else None+ )+ request_headers = _safe_exception_attribute(request, "headers") if request is not None else None+ client_request_id = _first_exception_attribute(chain, "request_id")+ if client_request_id is None:+ client_request_id = _first_header_value(request_headers, ("x-client-request-id",))++ method = _safe_exception_attribute(request, "method") if request is not None else None+ url = _safe_exception_attribute(request, "url") if request is not None else None+ # Query strings can contain credentials. The Ark API endpoint path is sufficient for diagnosis.+ safe_url = str(url).split("?", 1)[0] if url is not None else "-"++ causes = [f"{type(item).__name__}: {_redact_error_text(str(item))}" for item in chain[1:]]+ stack_frames: list[str] = []+ for item in chain:+ item_traceback = _safe_exception_attribute(item, "__traceback__")+ if item_traceback is None:+ continue+ for frame in traceback.extract_tb(item_traceback)[-12:]:+ stack_frames.append(+ f"{type(item).__name__}@{frame.filename}:{frame.lineno} in {frame.name}"+ )+ return {+ "error_type": type(error).__name__,+ "status": str(status) if status is not None else "-",+ "code": _render_error_log_value(_first_exception_attribute(chain, "code")),+ "client_request_id": _render_error_log_value(client_request_id),+ "server_request_id": _first_header_value(response_headers, _SERVER_REQUEST_ID_HEADERS)+ or "-",+ "server_trace_id": _first_header_value(response_headers, _SERVER_TRACE_ID_HEADERS) or "-",+ "method": str(method) if method is not None else "-",+ "url": safe_url,+ "response_body": _render_error_log_value(body),+ "cause_chain": " <- ".join(causes) if causes else "-",+ "traceback": " <- ".join(stack_frames) if stack_frames else "-",+ }+class VLMProviderAdapter(LLMProvider):"""Adapter that wraps an openviking VLMBase instance as an LLMProvider.@@ -119,6 +320,7 @@ class VLMProviderAdapter(LLMProvider):except Exception as e:if langfuse_observation:self._end_langfuse_observation_error(langfuse_observation, e)+ self._log_request_exception("chat", e, model=effective_model)return LLMResponse(content=f"Error calling LLM in VLM Adapter: {str(e)}",finish_reason="error",@@ -169,6 +371,7 @@ class VLMProviderAdapter(LLMProvider):):yield eventexcept Exception as exc:+ self._log_request_exception("chat_stream", exc, model=model or self._default_model)yield LLMStreamEvent(type="response",response=LLMResponse(@@ -177,6 +380,36 @@ class VLMProviderAdapter(LLMProvider):),)+ def _log_request_exception(+ self,+ operation: str,+ error: BaseException,+ *,+ model: str,+ ) -> None:+ details = _exception_log_details(error)+ logger.error(+ "VLM Adapter request failed: operation={} provider={} model={} timeout={}s "+ "error_type={} status={} code={} client_request_id={} "+ "server_request_id={} server_trace_id={} method={} url={} "+ "response_body={} cause_chain={} traceback={}",+ operation,+ getattr(self._vlm, "provider", None) or "unknown",+ model,+ getattr(self._vlm, "timeout", None) or "-",+ details["error_type"],+ details["status"],+ details["code"],+ details["client_request_id"],+ details["server_request_id"],+ details["server_trace_id"],+ details["method"],+ details["url"],+ details["response_body"],+ details["cause_chain"],+ details["traceback"],+ )+@classmethoddef _supports_native_stream(cls, vlm: Any) -> bool:instances = getattr(vlm, "_vlm_instances", None)@@ -379,9 +612,9 @@ class VLMProviderAdapter(LLMProvider):}if effective_max_tokens is not None:kwargs["max_tokens"] = effective_max_tokens- extra_headers = getattr(vlm, "extra_headers", None)- if extra_headers:- kwargs["extra_headers"] = extra_headers+ kwargs["extra_headers"] = build_volcengine_request_headers(+ getattr(vlm, "extra_headers", None)+ )if tools:kwargs["tools"] = toolskwargs["tool_choice"] = "auto"diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.pyindex 08b4ae3db..6134faff6 100644--- a/openviking/models/vlm/backends/volcengine_vlm.py+++ b/openviking/models/vlm/backends/volcengine_vlm.py@@ -6,6 +6,7 @@ import asyncioimport base64import jsonimport time+import uuidfrom pathlib import Pathfrom typing import Any, Dict, List, Optional, Union@@ -29,6 +30,24 @@ def _build_volcengine_headers(extra_headers: Optional[Dict[str, str]]) -> Dict[sreturn headers+def build_volcengine_request_headers(+ extra_headers: Optional[Dict[str, str]],+) -> Dict[str, str]:+ """Return per-request headers with a unique default client request ID.++ The existing prefix identifies OpenViking service traffic. A UUID suffix+ makes an individual Ark request searchable while custom client request ID+ values remain unchanged.+ """+ headers = _build_volcengine_headers(extra_headers)+ header_key = next(+ key for key in headers if key.lower() == VOLCENGINE_CLIENT_REQUEST_ID_HEADER.lower()+ )+ if headers[header_key] == VOLCENGINE_CLIENT_REQUEST_ID:+ headers[header_key] = f"{VOLCENGINE_CLIENT_REQUEST_ID},{uuid.uuid4().hex}"+ return headers++class VolcEngineVLM(OpenAIVLM):"""VolcEngine VLM backend with Chat Completions API support."""@@ -165,7 +184,7 @@ class VolcEngineVLM(OpenAIVLM):"messages": kwargs_messages,"temperature": self.temperature,"thinking": {"type": "disabled" if not effective_thinking else "enabled"},- "extra_headers": self.extra_headers,+ "extra_headers": build_volcengine_request_headers(self.extra_headers),}if self.max_tokens is not None:kwargs["max_tokens"] = self.max_tokens@@ -200,7 +219,7 @@ class VolcEngineVLM(OpenAIVLM):"messages": kwargs_messages,"temperature": self.temperature,"thinking": {"type": "disabled" if not effective_thinking else "enabled"},- "extra_headers": self.extra_headers,+ "extra_headers": build_volcengine_request_headers(self.extra_headers),}if self.max_tokens is not None:kwargs["max_tokens"] = self.max_tokens@@ -384,7 +403,7 @@ class VolcEngineVLM(OpenAIVLM):"messages": kwargs_messages,"temperature": self.temperature,"thinking": {"type": "disabled" if not effective_thinking else "enabled"},- "extra_headers": self.extra_headers,+ "extra_headers": build_volcengine_request_headers(self.extra_headers),}if self.max_tokens is not None:kwargs["max_tokens"] = self.max_tokens@@ -428,7 +447,7 @@ class VolcEngineVLM(OpenAIVLM):"messages": kwargs_messages,"temperature": self.temperature,"thinking": {"type": "disabled" if not effective_thinking else "enabled"},- "extra_headers": self.extra_headers,+ "extra_headers": build_volcengine_request_headers(self.extra_headers),}if self.max_tokens is not None:kwargs["max_tokens"] = self.max_tokensdiff --git a/tests/unit/test_vikingbot_vlm_adapter_retry.py b/tests/unit/test_vikingbot_vlm_adapter_retry.pyindex 44ee4a918..717ccc1ac 100644--- a/tests/unit/test_vikingbot_vlm_adapter_retry.py+++ b/tests/unit/test_vikingbot_vlm_adapter_retry.py@@ -4,9 +4,19 @@ import httpximport pytestimport vikingbot.providers.vlm_adapter as vlm_adapterfrom vikingbot.providers.vlm_adapter import VLMProviderAdapter-from volcenginesdkarkruntime._exceptions import ArkRateLimitError+from volcenginesdkarkruntime._exceptions import (+ ArkAPITimeoutError,+ ArkBadRequestError,+ ArkRateLimitError,+)from openviking.models.vlm.backends.openai_vlm import OpenAIVLM+from openviking.models.vlm.backends.volcengine_vlm import (+ VOLCENGINE_CLIENT_REQUEST_ID,+ VOLCENGINE_CLIENT_REQUEST_ID_HEADER,+ VolcEngineVLM,+ build_volcengine_request_headers,+)from openviking.utils.model_retry import is_retryable_rate_limit_error@@ -51,9 +61,11 @@ class _FakeStreamingCompletions:self.failures = list(failures)self.chunks = chunksself.calls = 0+ self.kwargs: list[dict] = []- async def create(self, **_kwargs):+ async def create(self, **kwargs):self.calls += 1+ self.kwargs.append(kwargs)if self.failures:raise self.failures.pop(0)return _AsyncChunks(self.chunks)@@ -73,6 +85,14 @@ class _FakeStreamingVLM:return self._client+class _CaptureLogger:+ def __init__(self):+ self.messages: list[str] = []++ def error(self, message: str, *args):+ self.messages.append(message.format(*args))++@pytest.mark.asyncioasync def test_chat_retries_rate_limit_until_success(monkeypatch):sleep_delays: list[float] = []@@ -255,6 +275,167 @@ async def test_chat_stream_routes_failover_wrapper_through_completion_api():assert events[0].response.content == "fallback-safe"+async def test_chat_stream_logs_timeout_cause_and_request_metadata(monkeypatch):+ request = httpx.Request(+ "POST",+ "https://ark.example.test/api/v3/chat/completions?api_key=must-not-leak",+ )+ cause = httpx.ReadTimeout("read deadline exceeded", request=request)+ error = ArkAPITimeoutError(request=request, request_id="request-123")+ error.__cause__ = cause+ completions = _FakeStreamingCompletions([error], [])+ adapter = VLMProviderAdapter(+ _FakeStreamingVLM(completions),+ "test-model",+ langfuse_client=_DisabledLangfuse(),+ )+ captured = _CaptureLogger()+ monkeypatch.setattr(vlm_adapter, "logger", captured)++ events = [+ event+ async for event in adapter.chat_stream(+ messages=[{"role": "user", "content": "hello"}],+ )+ ]++ assert events[-1].response.finish_reason == "error"+ assert len(captured.messages) == 1+ log_message = captured.messages[0]+ assert "operation=chat_stream" in log_message+ assert "error_type=ArkAPITimeoutError" in log_message+ assert 'client_request_id="request-123"' in log_message+ assert "server_request_id=-" in log_message+ assert "server_trace_id=-" in log_message+ assert "method=POST" in log_message+ assert "url=https://ark.example.test/api/v3/chat/completions" in log_message+ assert "cause_chain=ReadTimeout: read deadline exceeded" in log_message+ assert "must-not-leak" not in log_message+++def test_exception_log_details_include_redacted_upstream_error_body():+ request = httpx.Request(+ "POST",+ "https://ark.example.test/api/v3/chat/completions?token=query-secret",+ )+ response = httpx.Response(+ 400,+ request=request,+ headers={+ "X-Request-Id": "ark-server-request-456",+ "X-Trace-Id": "ark-server-trace-456",+ },+ )+ error = ArkBadRequestError(+ "invalid request",+ response=response,+ body={+ "code": "InvalidParameter",+ "message": "invalid max_tokens",+ "api_key": "body-secret",+ "authorization": "Bearer header-secret",+ },+ request_id="request-456",+ )++ details = vlm_adapter._exception_log_details(error)++ assert details == {+ "error_type": "ArkBadRequestError",+ "status": "400",+ "code": '"InvalidParameter"',+ "client_request_id": '"request-456"',+ "server_request_id": "ark-server-request-456",+ "server_trace_id": "ark-server-trace-456",+ "method": "POST",+ "url": "https://ark.example.test/api/v3/chat/completions",+ "response_body": (+ '{"code":"InvalidParameter","message":"invalid max_tokens",'+ '"api_key":"<redacted>","authorization":"<redacted>"}'+ ),+ "cause_chain": "-",+ "traceback": "-",+ }+++async def test_volcengine_stream_uses_unique_default_client_request_ids():+ completions = _FakeStreamingCompletions([], [])+ adapter = VLMProviderAdapter(+ _FakeStreamingVLM(completions),+ "test-model",+ langfuse_client=_DisabledLangfuse(),+ )++ for _ in range(2):+ _ = [+ event+ async for event in adapter.chat_stream(+ messages=[{"role": "user", "content": "hello"}],+ )+ ]++ request_ids = [+ kwargs["extra_headers"][VOLCENGINE_CLIENT_REQUEST_ID_HEADER]+ for kwargs in completions.kwargs+ ]+ assert len(set(request_ids)) == 2+ assert all(value.startswith(f"{VOLCENGINE_CLIENT_REQUEST_ID},") for value in request_ids)+++async def test_volcengine_non_stream_uses_unique_default_client_request_ids(monkeypatch):+ calls: list[dict] = []++ async def create(**kwargs):+ calls.append(kwargs)+ return SimpleNamespace(+ choices=[+ SimpleNamespace(+ message=SimpleNamespace(content="ok", tool_calls=None),+ finish_reason="stop",+ )+ ],+ usage=None,+ )++ vlm = VolcEngineVLM(+ {+ "provider": "volcengine",+ "model": "test-model",+ "api_key": "test-key",+ "max_retries": 0,+ }+ )+ monkeypatch.setattr(+ vlm,+ "get_async_client",+ lambda: SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))),+ )++ for _ in range(2):+ assert await vlm.get_completion_async(messages=[{"role": "user", "content": "hi"}]) == "ok"++ request_ids = [kwargs["extra_headers"][VOLCENGINE_CLIENT_REQUEST_ID_HEADER] for kwargs in calls]+ assert len(set(request_ids)) == 2+ assert all(value.startswith(f"{VOLCENGINE_CLIENT_REQUEST_ID},") for value in request_ids)+++def test_volcengine_request_headers_preserve_custom_client_request_id():+ headers = build_volcengine_request_headers(+ {+ "x-client-request-id": "caller-owned-request-id",+ "X-Tenant": "tenant-a",+ }+ )++ assert headers == {+ "x-client-request-id": "caller-owned-request-id",+ "X-Tenant": "tenant-a",+ }++def test_rate_limit_classifier_handles_target_error():assert is_retryable_rate_limit_error(RuntimeError("Error code: 429 - ModelAccountTpmRateLimitExceeded")