Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: improve Ark VLM error diagnostics (#4072)

DuTaof3202001f731
diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.py
index aaa8a09fa..9787bf491 100644
--- a/bot/vikingbot/providers/vlm_adapter.py
+++ b/bot/vikingbot/providers/vlm_adapter.py
@@ -6,12 +6,16 @@ configuration semantics are consistent with openviking server's vlm section.
"""
import asyncio
+import json
+import re
import time
-from collections.abc import AsyncIterator
+import traceback
+from collections.abc import AsyncIterator, Mapping
from typing import Any
from loguru import logger
+from openviking.models.vlm.backends.volcengine_vlm import build_volcengine_request_headers
from openviking.utils.model_retry import is_retryable_rate_limit_error, rate_limit_retry_delay
from openviking.utils.multimodal import redact_image_data_urls
from vikingbot.integrations.langfuse import LangfuseClient
@@ -26,6 +30,203 @@ from vikingbot.providers.base import (
)
from vikingbot.utils.tracing import get_current_response_id
+_MAX_ERROR_LOG_VALUE_CHARS = 4096
+_MAX_ERROR_LOG_ITEMS = 50
+_SENSITIVE_ERROR_KEYS = frozenset(
+ {
+ "access_token",
+ "api_key",
+ "apikey",
+ "authorization",
+ "client_secret",
+ "credential",
+ "credentials",
+ "id_token",
+ "password",
+ "refresh_token",
+ "secret",
+ "session_token",
+ "token",
+ }
+)
+_SENSITIVE_ERROR_KEY_SUFFIXES = (
+ "api_key",
+ "password",
+ "secret",
+ "token",
+)
+_SENSITIVE_ERROR_KV_RE = re.compile(
+ r"(?i)((?:api[_-]?key|authorization|credential|password|secret|token)"
+ r"[\"']?\s*[:=]\s*[\"']?)[^\s,;\]\}\"']+"
+)
+_BEARER_TOKEN_RE = re.compile(r"(?i)(bearer\s+)[^\s,;\]\}\"']+")
+_API_KEY_TOKEN_RE = re.compile(r"\bsk-[a-zA-Z0-9._-]{8,}")
+_SERVER_REQUEST_ID_HEADERS = ("x-request-id",)
+_SERVER_TRACE_ID_HEADERS = (
+ "x-trace-id",
+ "trace-id",
+ "x-tt-logid",
+ "x-log-id",
+)
+
+
+def _safe_exception_attribute(error: BaseException, name: str) -> Any:
+ try:
+ return getattr(error, name, None)
+ except BaseException:
+ return None
+
+
+def _exception_chain(error: BaseException) -> list[BaseException]:
+ chain: list[BaseException] = []
+ seen: set[int] = set()
+ current: BaseException | None = error
+ while current is not None and id(current) not in seen:
+ seen.add(id(current))
+ chain.append(current)
+ current = _safe_exception_attribute(current, "__cause__") or _safe_exception_attribute(
+ current, "__context__"
+ )
+ return chain
+
+
+def _redact_error_text(value: str) -> str:
+ value = redact_image_data_urls(value)
+ value = _SENSITIVE_ERROR_KV_RE.sub(lambda match: f"{match.group(1)}<redacted>", value)
+ value = _BEARER_TOKEN_RE.sub(lambda match: f"{match.group(1)}<redacted>", value)
+ value = _API_KEY_TOKEN_RE.sub("<redacted>", value)
+ if len(value) > _MAX_ERROR_LOG_VALUE_CHARS:
+ return value[:_MAX_ERROR_LOG_VALUE_CHARS] + "...[truncated]"
+ return value
+
+
+def _sanitize_error_value(value: Any, *, depth: int = 0) -> Any:
+ """Prepare an upstream error body for logs without exposing credentials or huge values."""
+ if depth >= 5:
+ return "<max-depth>"
+ if isinstance(value, Mapping):
+ sanitized: dict[str, Any] = {}
+ for index, (key, item) in enumerate(value.items()):
+ if index >= _MAX_ERROR_LOG_ITEMS:
+ sanitized["<truncated>"] = f"{len(value) - _MAX_ERROR_LOG_ITEMS} more items"
+ break
+ key_text = str(key)
+ normalized_key = key_text.lower().replace("-", "_")
+ if normalized_key in _SENSITIVE_ERROR_KEYS or any(
+ normalized_key.endswith(f"_{suffix}") for suffix in _SENSITIVE_ERROR_KEY_SUFFIXES
+ ):
+ sanitized[key_text] = "<redacted>"
+ else:
+ sanitized[key_text] = _sanitize_error_value(item, depth=depth + 1)
+ return sanitized
+ if isinstance(value, (list, tuple, set, frozenset)):
+ items = list(value)
+ sanitized_items = [
+ _sanitize_error_value(item, depth=depth + 1) for item in items[:_MAX_ERROR_LOG_ITEMS]
+ ]
+ if len(items) > _MAX_ERROR_LOG_ITEMS:
+ sanitized_items.append(f"<{len(items) - _MAX_ERROR_LOG_ITEMS} more items>")
+ return sanitized_items
+ if isinstance(value, bytes):
+ value = value.decode("utf-8", errors="replace")
+ if isinstance(value, str):
+ return _redact_error_text(value)
+ if value is None or isinstance(value, (bool, int, float)):
+ return value
+ try:
+ return _redact_error_text(str(value))
+ except BaseException:
+ return f"<{type(value).__name__}>"
+
+
+def _render_error_log_value(value: Any) -> str:
+ if value is None:
+ return "-"
+ sanitized = _sanitize_error_value(value)
+ try:
+ rendered = json.dumps(sanitized, ensure_ascii=False, default=str, separators=(",", ":"))
+ except BaseException:
+ rendered = _redact_error_text(str(sanitized))
+ if len(rendered) > _MAX_ERROR_LOG_VALUE_CHARS:
+ return rendered[:_MAX_ERROR_LOG_VALUE_CHARS] + "...[truncated]"
+ return rendered
+
+
+def _first_exception_attribute(chain: list[BaseException], name: str) -> Any:
+ for item in chain:
+ value = _safe_exception_attribute(item, name)
+ if value is not None:
+ return value
+ return None
+
+
+def _first_header_value(headers: Any, names: tuple[str, ...]) -> str | None:
+ if headers is None:
+ return None
+ for name in names:
+ try:
+ value = headers.get(name)
+ except BaseException:
+ value = None
+ if value:
+ return _redact_error_text(str(value))
+ return None
+
+
+def _exception_log_details(error: BaseException) -> dict[str, str]:
+ """Extract Ark/OpenAI-compatible exception details for one structured log line."""
+ chain = _exception_chain(error)
+ response = _first_exception_attribute(chain, "response")
+ request = _first_exception_attribute(chain, "request")
+ if request is None and response is not None:
+ request = _safe_exception_attribute(response, "request")
+
+ status = _first_exception_attribute(chain, "status_code")
+ if status is None and response is not None:
+ status = _safe_exception_attribute(response, "status_code")
+
+ body = _first_exception_attribute(chain, "body")
+ if body is None and response is not None:
+ body = _safe_exception_attribute(response, "text")
+
+ response_headers = (
+ _safe_exception_attribute(response, "headers") if response is not None else None
+ )
+ request_headers = _safe_exception_attribute(request, "headers") if request is not None else None
+ client_request_id = _first_exception_attribute(chain, "request_id")
+ if client_request_id is None:
+ client_request_id = _first_header_value(request_headers, ("x-client-request-id",))
+
+ method = _safe_exception_attribute(request, "method") if request is not None else None
+ url = _safe_exception_attribute(request, "url") if request is not None else None
+ # Query strings can contain credentials. The Ark API endpoint path is sufficient for diagnosis.
+ safe_url = str(url).split("?", 1)[0] if url is not None else "-"
+
+ causes = [f"{type(item).__name__}: {_redact_error_text(str(item))}" for item in chain[1:]]
+ stack_frames: list[str] = []
+ for item in chain:
+ item_traceback = _safe_exception_attribute(item, "__traceback__")
+ if item_traceback is None:
+ continue
+ for frame in traceback.extract_tb(item_traceback)[-12:]:
+ stack_frames.append(
+ f"{type(item).__name__}@{frame.filename}:{frame.lineno} in {frame.name}"
+ )
+ return {
+ "error_type": type(error).__name__,
+ "status": str(status) if status is not None else "-",
+ "code": _render_error_log_value(_first_exception_attribute(chain, "code")),
+ "client_request_id": _render_error_log_value(client_request_id),
+ "server_request_id": _first_header_value(response_headers, _SERVER_REQUEST_ID_HEADERS)
+ or "-",
+ "server_trace_id": _first_header_value(response_headers, _SERVER_TRACE_ID_HEADERS) or "-",
+ "method": str(method) if method is not None else "-",
+ "url": safe_url,
+ "response_body": _render_error_log_value(body),
+ "cause_chain": " <- ".join(causes) if causes else "-",
+ "traceback": " <- ".join(stack_frames) if stack_frames else "-",
+ }
+
class VLMProviderAdapter(LLMProvider):
"""Adapter that wraps an openviking VLMBase instance as an LLMProvider.
@@ -119,6 +320,7 @@ class VLMProviderAdapter(LLMProvider):
except Exception as e:
if langfuse_observation:
self._end_langfuse_observation_error(langfuse_observation, e)
+ self._log_request_exception("chat", e, model=effective_model)
return LLMResponse(
content=f"Error calling LLM in VLM Adapter: {str(e)}",
finish_reason="error",
@@ -169,6 +371,7 @@ class VLMProviderAdapter(LLMProvider):
):
yield event
except Exception as exc:
+ self._log_request_exception("chat_stream", exc, model=model or self._default_model)
yield LLMStreamEvent(
type="response",
response=LLMResponse(
@@ -177,6 +380,36 @@ class VLMProviderAdapter(LLMProvider):
),
)
+ def _log_request_exception(
+ self,
+ operation: str,
+ error: BaseException,
+ *,
+ model: str,
+ ) -> None:
+ details = _exception_log_details(error)
+ logger.error(
+ "VLM Adapter request failed: operation={} provider={} model={} timeout={}s "
+ "error_type={} status={} code={} client_request_id={} "
+ "server_request_id={} server_trace_id={} method={} url={} "
+ "response_body={} cause_chain={} traceback={}",
+ operation,
+ getattr(self._vlm, "provider", None) or "unknown",
+ model,
+ getattr(self._vlm, "timeout", None) or "-",
+ details["error_type"],
+ details["status"],
+ details["code"],
+ details["client_request_id"],
+ details["server_request_id"],
+ details["server_trace_id"],
+ details["method"],
+ details["url"],
+ details["response_body"],
+ details["cause_chain"],
+ details["traceback"],
+ )
+
@classmethod
def _supports_native_stream(cls, vlm: Any) -> bool:
instances = getattr(vlm, "_vlm_instances", None)
@@ -379,9 +612,9 @@ class VLMProviderAdapter(LLMProvider):
}
if effective_max_tokens is not None:
kwargs["max_tokens"] = effective_max_tokens
- extra_headers = getattr(vlm, "extra_headers", None)
- if extra_headers:
- kwargs["extra_headers"] = extra_headers
+ kwargs["extra_headers"] = build_volcengine_request_headers(
+ getattr(vlm, "extra_headers", None)
+ )
if tools:
kwargs["tools"] = tools
kwargs["tool_choice"] = "auto"
diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.py
index 08b4ae3db..6134faff6 100644
--- a/openviking/models/vlm/backends/volcengine_vlm.py
+++ b/openviking/models/vlm/backends/volcengine_vlm.py
@@ -6,6 +6,7 @@ import asyncio
import base64
import json
import time
+import uuid
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
@@ -29,6 +30,24 @@ def _build_volcengine_headers(extra_headers: Optional[Dict[str, str]]) -> Dict[s
return headers
+def build_volcengine_request_headers(
+ extra_headers: Optional[Dict[str, str]],
+) -> Dict[str, str]:
+ """Return per-request headers with a unique default client request ID.
+
+ The existing prefix identifies OpenViking service traffic. A UUID suffix
+ makes an individual Ark request searchable while custom client request ID
+ values remain unchanged.
+ """
+ headers = _build_volcengine_headers(extra_headers)
+ header_key = next(
+ key for key in headers if key.lower() == VOLCENGINE_CLIENT_REQUEST_ID_HEADER.lower()
+ )
+ if headers[header_key] == VOLCENGINE_CLIENT_REQUEST_ID:
+ headers[header_key] = f"{VOLCENGINE_CLIENT_REQUEST_ID},{uuid.uuid4().hex}"
+ return headers
+
+
class VolcEngineVLM(OpenAIVLM):
"""VolcEngine VLM backend with Chat Completions API support."""
@@ -165,7 +184,7 @@ class VolcEngineVLM(OpenAIVLM):
"messages": kwargs_messages,
"temperature": self.temperature,
"thinking": {"type": "disabled" if not effective_thinking else "enabled"},
- "extra_headers": self.extra_headers,
+ "extra_headers": build_volcengine_request_headers(self.extra_headers),
}
if self.max_tokens is not None:
kwargs["max_tokens"] = self.max_tokens
@@ -200,7 +219,7 @@ class VolcEngineVLM(OpenAIVLM):
"messages": kwargs_messages,
"temperature": self.temperature,
"thinking": {"type": "disabled" if not effective_thinking else "enabled"},
- "extra_headers": self.extra_headers,
+ "extra_headers": build_volcengine_request_headers(self.extra_headers),
}
if self.max_tokens is not None:
kwargs["max_tokens"] = self.max_tokens
@@ -384,7 +403,7 @@ class VolcEngineVLM(OpenAIVLM):
"messages": kwargs_messages,
"temperature": self.temperature,
"thinking": {"type": "disabled" if not effective_thinking else "enabled"},
- "extra_headers": self.extra_headers,
+ "extra_headers": build_volcengine_request_headers(self.extra_headers),
}
if self.max_tokens is not None:
kwargs["max_tokens"] = self.max_tokens
@@ -428,7 +447,7 @@ class VolcEngineVLM(OpenAIVLM):
"messages": kwargs_messages,
"temperature": self.temperature,
"thinking": {"type": "disabled" if not effective_thinking else "enabled"},
- "extra_headers": self.extra_headers,
+ "extra_headers": build_volcengine_request_headers(self.extra_headers),
}
if self.max_tokens is not None:
kwargs["max_tokens"] = self.max_tokens
diff --git a/tests/unit/test_vikingbot_vlm_adapter_retry.py b/tests/unit/test_vikingbot_vlm_adapter_retry.py
index 44ee4a918..717ccc1ac 100644
--- a/tests/unit/test_vikingbot_vlm_adapter_retry.py
+++ b/tests/unit/test_vikingbot_vlm_adapter_retry.py
@@ -4,9 +4,19 @@ import httpx
import pytest
import vikingbot.providers.vlm_adapter as vlm_adapter
from vikingbot.providers.vlm_adapter import VLMProviderAdapter
-from volcenginesdkarkruntime._exceptions import ArkRateLimitError
+from volcenginesdkarkruntime._exceptions import (
+ ArkAPITimeoutError,
+ ArkBadRequestError,
+ ArkRateLimitError,
+)
from openviking.models.vlm.backends.openai_vlm import OpenAIVLM
+from openviking.models.vlm.backends.volcengine_vlm import (
+ VOLCENGINE_CLIENT_REQUEST_ID,
+ VOLCENGINE_CLIENT_REQUEST_ID_HEADER,
+ VolcEngineVLM,
+ build_volcengine_request_headers,
+)
from openviking.utils.model_retry import is_retryable_rate_limit_error
@@ -51,9 +61,11 @@ class _FakeStreamingCompletions:
self.failures = list(failures)
self.chunks = chunks
self.calls = 0
+ self.kwargs: list[dict] = []
- async def create(self, **_kwargs):
+ async def create(self, **kwargs):
self.calls += 1
+ self.kwargs.append(kwargs)
if self.failures:
raise self.failures.pop(0)
return _AsyncChunks(self.chunks)
@@ -73,6 +85,14 @@ class _FakeStreamingVLM:
return self._client
+class _CaptureLogger:
+ def __init__(self):
+ self.messages: list[str] = []
+
+ def error(self, message: str, *args):
+ self.messages.append(message.format(*args))
+
+
@pytest.mark.asyncio
async def test_chat_retries_rate_limit_until_success(monkeypatch):
sleep_delays: list[float] = []
@@ -255,6 +275,167 @@ async def test_chat_stream_routes_failover_wrapper_through_completion_api():
assert events[0].response.content == "fallback-safe"
+async def test_chat_stream_logs_timeout_cause_and_request_metadata(monkeypatch):
+ request = httpx.Request(
+ "POST",
+ "https://ark.example.test/api/v3/chat/completions?api_key=must-not-leak",
+ )
+ cause = httpx.ReadTimeout("read deadline exceeded", request=request)
+ error = ArkAPITimeoutError(request=request, request_id="request-123")
+ error.__cause__ = cause
+ completions = _FakeStreamingCompletions([error], [])
+ adapter = VLMProviderAdapter(
+ _FakeStreamingVLM(completions),
+ "test-model",
+ langfuse_client=_DisabledLangfuse(),
+ )
+ captured = _CaptureLogger()
+ monkeypatch.setattr(vlm_adapter, "logger", captured)
+
+ events = [
+ event
+ async for event in adapter.chat_stream(
+ messages=[{"role": "user", "content": "hello"}],
+ )
+ ]
+
+ assert events[-1].response.finish_reason == "error"
+ assert len(captured.messages) == 1
+ log_message = captured.messages[0]
+ assert "operation=chat_stream" in log_message
+ assert "error_type=ArkAPITimeoutError" in log_message
+ assert 'client_request_id="request-123"' in log_message
+ assert "server_request_id=-" in log_message
+ assert "server_trace_id=-" in log_message
+ assert "method=POST" in log_message
+ assert "url=https://ark.example.test/api/v3/chat/completions" in log_message
+ assert "cause_chain=ReadTimeout: read deadline exceeded" in log_message
+ assert "must-not-leak" not in log_message
+
+
+def test_exception_log_details_include_redacted_upstream_error_body():
+ request = httpx.Request(
+ "POST",
+ "https://ark.example.test/api/v3/chat/completions?token=query-secret",
+ )
+ response = httpx.Response(
+ 400,
+ request=request,
+ headers={
+ "X-Request-Id": "ark-server-request-456",
+ "X-Trace-Id": "ark-server-trace-456",
+ },
+ )
+ error = ArkBadRequestError(
+ "invalid request",
+ response=response,
+ body={
+ "code": "InvalidParameter",
+ "message": "invalid max_tokens",
+ "api_key": "body-secret",
+ "authorization": "Bearer header-secret",
+ },
+ request_id="request-456",
+ )
+
+ details = vlm_adapter._exception_log_details(error)
+
+ assert details == {
+ "error_type": "ArkBadRequestError",
+ "status": "400",
+ "code": '"InvalidParameter"',
+ "client_request_id": '"request-456"',
+ "server_request_id": "ark-server-request-456",
+ "server_trace_id": "ark-server-trace-456",
+ "method": "POST",
+ "url": "https://ark.example.test/api/v3/chat/completions",
+ "response_body": (
+ '{"code":"InvalidParameter","message":"invalid max_tokens",'
+ '"api_key":"<redacted>","authorization":"<redacted>"}'
+ ),
+ "cause_chain": "-",
+ "traceback": "-",
+ }
+
+
+async def test_volcengine_stream_uses_unique_default_client_request_ids():
+ completions = _FakeStreamingCompletions([], [])
+ adapter = VLMProviderAdapter(
+ _FakeStreamingVLM(completions),
+ "test-model",
+ langfuse_client=_DisabledLangfuse(),
+ )
+
+ for _ in range(2):
+ _ = [
+ event
+ async for event in adapter.chat_stream(
+ messages=[{"role": "user", "content": "hello"}],
+ )
+ ]
+
+ request_ids = [
+ kwargs["extra_headers"][VOLCENGINE_CLIENT_REQUEST_ID_HEADER]
+ for kwargs in completions.kwargs
+ ]
+ assert len(set(request_ids)) == 2
+ assert all(value.startswith(f"{VOLCENGINE_CLIENT_REQUEST_ID},") for value in request_ids)
+
+
+async def test_volcengine_non_stream_uses_unique_default_client_request_ids(monkeypatch):
+ calls: list[dict] = []
+
+ async def create(**kwargs):
+ calls.append(kwargs)
+ return SimpleNamespace(
+ choices=[
+ SimpleNamespace(
+ message=SimpleNamespace(content="ok", tool_calls=None),
+ finish_reason="stop",
+ )
+ ],
+ usage=None,
+ )
+
+ vlm = VolcEngineVLM(
+ {
+ "provider": "volcengine",
+ "model": "test-model",
+ "api_key": "test-key",
+ "max_retries": 0,
+ }
+ )
+ monkeypatch.setattr(
+ vlm,
+ "get_async_client",
+ lambda: SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create))),
+ )
+
+ for _ in range(2):
+ assert await vlm.get_completion_async(messages=[{"role": "user", "content": "hi"}]) == "ok"
+
+ request_ids = [kwargs["extra_headers"][VOLCENGINE_CLIENT_REQUEST_ID_HEADER] for kwargs in calls]
+ assert len(set(request_ids)) == 2
+ assert all(value.startswith(f"{VOLCENGINE_CLIENT_REQUEST_ID},") for value in request_ids)
+
+
+def test_volcengine_request_headers_preserve_custom_client_request_id():
+ headers = build_volcengine_request_headers(
+ {
+ "x-client-request-id": "caller-owned-request-id",
+ "X-Tenant": "tenant-a",
+ }
+ )
+
+ assert headers == {
+ "x-client-request-id": "caller-owned-request-id",
+ "X-Tenant": "tenant-a",
+ }
+
+
def test_rate_limit_classifier_handles_target_error():
assert is_retryable_rate_limit_error(
RuntimeError("Error code: 429 - ModelAccountTpmRateLimitExceeded")