Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(bot): support image inputs in VikingBot Chat API (#3619)
DuTao6b416cd66d54
diff --git a/bot/vikingbot/agent/context.py b/bot/vikingbot/agent/context.pyindex 854537b83..ac047d2a8 100644--- a/bot/vikingbot/agent/context.py+++ b/bot/vikingbot/agent/context.py@@ -347,7 +347,7 @@ IMPORTANT:self,history: list[dict[str, Any]],current_message: str,- media: list[str] | None = None,+ media: list[str | dict[str, Any]] | None = None,session_key: SessionKey | None = None,ov_tools_enable: bool = True,profile_user_list: list[str] | None = None,@@ -442,13 +442,41 @@ IMPORTANT:return messages- def _build_user_content(self, text: str, media: list[str] | None) -> str | list[dict[str, Any]]:- """Build user message content with optional base64-encoded images."""+ def _build_user_content(+ self,+ text: str,+ media: list[str | dict[str, Any]] | None,+ ) -> str | list[dict[str, Any]]:+ """Build user content from trusted local paths or validated image URL parts."""if not media:return textimages = []- for path in media:+ for item in media:+ if isinstance(item, dict):+ image_url = item.get("image_url")+ url = image_url.get("url") if isinstance(image_url, dict) else None+ if item.get("type") == "image_url" and isinstance(url, str):+ images.append(+ {+ "type": "image_url",+ "image_url": {+ "url": url,+ **(+ {"detail": image_url["detail"]}+ if image_url.get("detail")+ else {}+ ),+ },+ }+ )+ continue++ if item.startswith(("https://", "data:image/")):+ images.append({"type": "image_url", "image_url": {"url": item}})+ continue++ path = itemp = Path(path)mime, _ = mimetypes.guess_type(path)if not p.is_file() or not mime or not mime.startswith("image/"):diff --git a/bot/vikingbot/bus/events.py b/bot/vikingbot/bus/events.pyindex 02da108b2..9492634d5 100644--- a/bot/vikingbot/bus/events.py+++ b/bot/vikingbot/bus/events.py@@ -37,7 +37,7 @@ class InboundMessage:timestamp: datetime = field(default_factory=datetime.now)sender_name: str | None = Noneneed_reply: bool = True- media: list[str] = field(default_factory=list) # Media URLs+ media: list[str | dict[str, Any]] = field(default_factory=list) # Paths or image_url partsmetadata: dict[str, Any] = field(default_factory=dict) # Channel-specific dataopenviking_connection: dict[str, Any] | None = None # Internal OpenViking identitydiff --git a/bot/vikingbot/channels/openapi.py b/bot/vikingbot/channels/openapi.pyindex a9d1434ef..553a86be7 100644--- a/bot/vikingbot/channels/openapi.py+++ b/bot/vikingbot/channels/openapi.py@@ -13,7 +13,8 @@ from urllib.parse import urlsplitimport httpxfrom fastapi import APIRouter, Depends, FastAPI, Header, HTTPException, Request-from fastapi.responses import Response, StreamingResponse+from fastapi.exceptions import RequestValidationError+from fastapi.responses import JSONResponse, Response, StreamingResponsefrom loguru import loggerfrom vikingbot.bus.events import InboundMessage, OutboundEventType, OutboundMessage@@ -66,6 +67,35 @@ HOP_BY_HOP_HEADERS = {}+def _public_validation_error(error: Any) -> dict[str, Any]:+ """Keep validation diagnostics without reflecting request data."""+ if not isinstance(error, dict):+ return {+ "type": "value_error",+ "loc": ["request"],+ "msg": "Invalid request value",+ }++ location = error.get("loc", ("request",))+ if not isinstance(location, (list, tuple)):+ location = (location,)+ return {+ "type": str(error.get("type") or "value_error"),+ "loc": [item if isinstance(item, (str, int)) else str(item) for item in location],+ "msg": str(error.get("msg") or "Invalid request value"),+ }+++async def _request_validation_error_handler(+ _request: Request,+ exc: RequestValidationError,+) -> JSONResponse:+ # Pydantic validation errors include the original `input` by default. Never+ # return it here: image inputs may contain large or sensitive Base64 data.+ errors = [_public_validation_error(error) for error in exc.errors()]+ return JSONResponse(status_code=422, content={"detail": errors})++@dataclass(frozen=True)class GatewayRequestAuth:"""Resolved gateway-level auth facts for one HTTP request."""@@ -998,9 +1028,7 @@ class OpenAPIChannel(BaseChannel):detail=OPENVIKING_UPSTREAM_NOT_CONFIGURED_DETAIL,)await self._assert_runtime_upstream_auth_mode(- self._identity_headers_from_connection(- compile_request.openviking_connection- )+ self._identity_headers_from_connection(compile_request.openviking_connection))compile_request._principal_scope = self._connection_principal_scope(compile_request.openviking_connection@@ -1200,6 +1228,11 @@ class OpenAPIChannel(BaseChannel):logger.warning("No external FastAPI app provided, cannot setup routes")return+ self._app.add_exception_handler(+ RequestValidationError,+ _request_validation_error_handler,+ )+# Get the router and include it at root path# Note: openviking-server adds its own /bot/v1 prefix when proxyingrouter = self.get_router()@@ -1220,6 +1253,12 @@ class OpenAPIChannel(BaseChannel):disabled_tools.append("message")return {"disabled_tools": disabled_tools}+ def _request_media(self, request: ChatRequest) -> list[dict[str, Any]]:+ return [image.model_dump(mode="json", exclude_none=True) for image in request.images]++ def _request_content(self, request: ChatRequest) -> str:+ return request.message or "[Image]"+def _request_openviking_connection(self, request: ChatRequest) -> dict[str, Any] | None:if request.openviking_connection:connection = request.openviking_connection.model_dump(exclude_none=True)@@ -1266,7 +1305,7 @@ class OpenAPIChannel(BaseChannel):)# Build content with context if provided- content = request.message+ content = self._request_content(request)if request.context:# Context is handled separately by session managerpass@@ -1277,6 +1316,7 @@ class OpenAPIChannel(BaseChannel):sender_id=user_id,actor_peer_id=self._request_actor_peer_id(request, user_id),content=content,+ media=self._request_media(request),metadata=self._request_metadata(request),openviking_connection=self._request_openviking_connection(request),)@@ -1347,7 +1387,8 @@ class OpenAPIChannel(BaseChannel):session_key=session_key,sender_id=user_id,actor_peer_id=self._request_actor_peer_id(request, user_id),- content=request.message,+ content=self._request_content(request),+ media=self._request_media(request),metadata=self._request_metadata(request),openviking_connection=self._request_openviking_connection(request),)@@ -1421,7 +1462,7 @@ class OpenAPIChannel(BaseChannel):)# Build content with context if provided- content = request.message+ content = self._request_content(request)if request.context:# Context is handled separately by session managerpass@@ -1433,6 +1474,7 @@ class OpenAPIChannel(BaseChannel):actor_peer_id=self._request_actor_peer_id(request, user_id),content=content,need_reply=request.need_reply,+ media=self._request_media(request),metadata=self._request_metadata(request),openviking_connection=self._request_openviking_connection(request),)@@ -1510,7 +1552,8 @@ class OpenAPIChannel(BaseChannel):session_key=session_key,sender_id=user_id,actor_peer_id=self._request_actor_peer_id(request, user_id),- content=request.message,+ content=self._request_content(request),+ media=self._request_media(request),metadata=self._request_metadata(request),openviking_connection=self._request_openviking_connection(request),)diff --git a/bot/vikingbot/channels/openapi_models.py b/bot/vikingbot/channels/openapi_models.pyindex 4abd9b34b..1ac16855a 100644--- a/bot/vikingbot/channels/openapi_models.py+++ b/bot/vikingbot/channels/openapi_models.py@@ -1,10 +1,78 @@"""Pydantic models for OpenAPI channel."""+import base64+import binasciifrom datetime import datetimefrom enum import Enum-from typing import Any, Dict, List, Optional--from pydantic import BaseModel, Field, PrivateAttr, model_validator+from typing import Any, Dict, List, Literal, Optional+from urllib.parse import urlsplit++from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator, model_validator++MAX_CHAT_IMAGES = 4+MAX_CHAT_INLINE_IMAGE_BYTES = 10 * 1024 * 1024+SUPPORTED_CHAT_INLINE_IMAGE_MIME_TYPES = {+ "image/gif",+ "image/jpeg",+ "image/png",+ "image/webp",+}+++def _detect_chat_image_mime(data: bytes) -> str | None:+ if data.startswith(b"\x89PNG\r\n\x1a\n"):+ return "image/png"+ if data.startswith(b"\xff\xd8\xff"):+ return "image/jpeg"+ if data.startswith((b"GIF87a", b"GIF89a")):+ return "image/gif"+ if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":+ return "image/webp"+ return None+++def _validate_chat_image_url(value: str) -> str:+ if value.startswith("https://"):+ parsed = urlsplit(value)+ if not parsed.hostname or parsed.username or parsed.password:+ raise ValueError("image URL must be an absolute HTTPS URL without credentials")+ # Match multimodal model APIs: pass remote URLs through without fetching+ # them. Content-type and format support are therefore provider-specific.+ return value++ if not value.startswith("data:image/"):+ raise ValueError("image URL must use HTTPS or an image Base64 data URL")++ try:+ header, encoded = value.split(",", 1)+ media_type, encoding = header[5:].split(";", 1)+ except ValueError as exc:+ raise ValueError("invalid image data URL") from exc++ media_type = media_type.lower()+ if media_type not in SUPPORTED_CHAT_INLINE_IMAGE_MIME_TYPES:+ raise ValueError("unsupported image MIME type; expected JPEG, PNG, GIF, or WebP")+ if encoding.lower() != "base64" or not encoded:+ raise ValueError("image data URL must contain Base64-encoded data")++ # Reject oversized payloads before decoding to avoid a large temporary allocation.+ if len(encoded) > ((MAX_CHAT_INLINE_IMAGE_BYTES + 2) // 3) * 4:+ raise ValueError(f"decoded image exceeds {MAX_CHAT_INLINE_IMAGE_BYTES} bytes")+ try:+ decoded = base64.b64decode(encoded, validate=True)+ except (binascii.Error, ValueError) as exc:+ raise ValueError("image data URL contains invalid Base64") from exc+ if len(decoded) > MAX_CHAT_INLINE_IMAGE_BYTES:+ raise ValueError(f"decoded image exceeds {MAX_CHAT_INLINE_IMAGE_BYTES} bytes")++ detected_type = _detect_chat_image_mime(decoded)+ if detected_type is None:+ raise ValueError("image data does not have a supported image signature")+ if detected_type != media_type:+ raise ValueError(+ f"image MIME type {media_type} does not match detected type {detected_type}"+ )+ return valueclass MessageRole(str, Enum):@@ -46,6 +114,35 @@ class ChatMessage(BaseModel):)+class ChatImageURL(BaseModel):+ """OpenAI-compatible image URL descriptor."""++ model_config = ConfigDict(extra="forbid")++ url: str = Field(+ ...,+ description="HTTPS passthrough URL or validated Base64 image data URL",+ )+ detail: Optional[Literal["auto", "low", "high"]] = Field(+ default=None,+ description="Optional image detail level; omitted for maximum provider compatibility",+ )++ @field_validator("url")+ @classmethod+ def validate_url(cls, value: str) -> str:+ return _validate_chat_image_url(value)+++class ChatImage(BaseModel):+ """OpenAI-compatible image input part."""++ model_config = ConfigDict(extra="forbid")++ type: Literal["image_url"] = "image_url"+ image_url: ChatImageURL++class OpenVikingConnection(BaseModel):"""OpenViking identity forwarded by the Studio proxy."""@@ -66,7 +163,12 @@ class OpenVikingConnection(BaseModel):class ChatRequest(BaseModel):"""Request body for chat endpoint."""- message: str = Field(..., description="User message to send", min_length=1)+ message: str = Field(default="", description="User message to send")+ images: List[ChatImage] = Field(+ default_factory=list,+ max_length=MAX_CHAT_IMAGES,+ description="HTTPS passthrough URLs or validated Base64 image inputs",+ )session_id: Optional[str] = Field(default=None, description="Session ID (optional, will create new if not provided)")@@ -89,6 +191,12 @@ class ChatRequest(BaseModel):description="Authenticated OpenViking connection forwarded by the server proxy",)+ @model_validator(mode="after")+ def validate_message_or_images(self) -> "ChatRequest":+ if not self.message.strip() and not self.images:+ raise ValueError("message or at least one image is required")+ return self+class ChatResponse(BaseModel):"""Response from chat endpoint (non-streaming)."""diff --git a/bot/vikingbot/cli/commands.py b/bot/vikingbot/cli/commands.pyindex 29f48ce3c..7c8224375 100644--- a/bot/vikingbot/cli/commands.py+++ b/bot/vikingbot/cli/commands.py@@ -310,8 +310,6 @@ def _make_provider(config, langfuse_client: Any = None):Bot's own VLM configuration. Otherwise the complete root VLM configurationis inherited, including its ordered credentials and failover behavior."""- from vikingbot.providers.litellm_provider import LiteLLMProvider-p = config.agentsmodel = p.model if p else Nonetemperature = p.temperature if p else 0.7@@ -431,20 +429,10 @@ def _make_provider(config, langfuse_client: Any = None):langfuse_client=langfuse_client,)- # Fallback: legacy LiteLLMProvider (no explicit provider set)- if not api_key and not model.startswith("bedrock/"):- console.print("[yellow]Warning: No API key configured.[/yellow]")- console.print("You can configure providers later in the Console UI.")-- return LiteLLMProvider(- api_key=api_key,- api_base=api_base,- default_model=model,- extra_headers=extra_headers,- provider_name=provider_name,- timeout=timeout,- thinking=thinking,- langfuse_client=langfuse_client,+ raise RuntimeError(+ "No VLM provider configured for VikingBot. Set bot.agents.provider, "+ "configure bot.agents.credentials, or inherit the root vlm configuration. "+ "Set provider to 'litellm' to use LiteLLM.")diff --git a/bot/vikingbot/providers/litellm_provider.py b/bot/vikingbot/providers/litellm_provider.pyindex 9ae711758..35a35e5ed 100644--- a/bot/vikingbot/providers/litellm_provider.py+++ b/bot/vikingbot/providers/litellm_provider.py@@ -7,6 +7,7 @@ import litellmfrom litellm import acompletionfrom loguru import logger+from openviking.utils.multimodal import redact_image_data_urlsfrom vikingbot.integrations.langfuse import LangfuseClientfrom vikingbot.providers.base import (LLMProvider,@@ -136,9 +137,7 @@ class LiteLLMProvider(LLMProvider):kwargs["thinking"] = {"type": "enabled"}return- if thinking_param == "dashscope_enable_thinking" or model_lower.startswith(- "dashscope/"- ):+ if thinking_param == "dashscope_enable_thinking" or model_lower.startswith("dashscope/"):extra_body = dict(kwargs.get("extra_body") or {})extra_body["enable_thinking"] = Truekwargs["extra_body"] = extra_body@@ -283,7 +282,7 @@ class LiteLLMProvider(LLMProvider):name="llm-chat",as_type="generation",model=model,- input=messages,+ input=redact_image_data_urls(messages),metadata=metadata,)if response_id:diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.pyindex de903eada..5ec814c85 100644--- a/bot/vikingbot/providers/vlm_adapter.py+++ b/bot/vikingbot/providers/vlm_adapter.py@@ -13,6 +13,7 @@ from typing import Anyfrom loguru import loggerfrom openviking.utils.model_retry import is_retryable_rate_limit_error, rate_limit_retry_delay+from openviking.utils.multimodal import redact_image_data_urlsfrom vikingbot.integrations.langfuse import LangfuseClientfrom vikingbot.providers.base import (LLMProvider,@@ -72,7 +73,7 @@ class VLMProviderAdapter(LLMProvider):name="llm-chat",as_type="generation",model=effective_model,- input=messages,+ input=redact_image_data_urls(messages),metadata=metadata,)if response_id:diff --git a/bot/vikingbot/tests/unit/test_bot_vlm_credentials.py b/bot/vikingbot/tests/unit/test_bot_vlm_credentials.pyindex 6e4527fbe..fd1277640 100644--- a/bot/vikingbot/tests/unit/test_bot_vlm_credentials.py+++ b/bot/vikingbot/tests/unit/test_bot_vlm_credentials.py@@ -5,6 +5,7 @@ from unittest.mock import AsyncMockimport pytestfrom openviking.models.vlm import MultiCredentialVLM+from openviking.models.vlm.backends.litellm_vlm import LiteLLMVLMProviderfrom vikingbot.config import loaderfrom vikingbot.config.schema import Configfrom vikingbot.providers.vlm_adapter import VLMProviderAdapter@@ -238,6 +239,50 @@ def test_explicit_bot_model_without_credentials_keeps_single_model_behavior(tmp_assert provider._vlm.model == "bot-model"+def test_explicit_litellm_provider_uses_vlm_adapter(tmp_path, monkeypatch):+ config = _write_config(+ tmp_path,+ monkeypatch,+ {+ "bot": {+ "agents": {+ "provider": "litellm",+ "model": "openrouter/openai/gpt-4o-mini",+ "api_key": "bot-key",+ }+ }+ },+ )++ from vikingbot.cli.commands import _make_provider++ provider = _make_provider(config)++ assert isinstance(provider, VLMProviderAdapter)+ assert isinstance(provider._vlm, LiteLLMVLMProvider)+ assert provider._vlm.model == "openrouter/openai/gpt-4o-mini"+++def test_bot_model_without_provider_rejects_legacy_fallback(tmp_path, monkeypatch):+ config = _write_config(+ tmp_path,+ monkeypatch,+ {+ "bot": {+ "agents": {+ "model": "openrouter/openai/gpt-4o-mini",+ "api_key": "bot-key",+ }+ }+ },+ )++ from vikingbot.cli.commands import _make_provider++ with pytest.raises(RuntimeError, match="Set provider to 'litellm'"):+ _make_provider(config)++def test_saving_inherited_config_does_not_turn_root_model_into_bot_override(tmp_path, monkeypatch):config = _write_config(tmp_path,diff --git a/docs/en/api/24-vikingbot.md b/docs/en/api/24-vikingbot.mdindex 88b3985d8..3bbe81007 100644--- a/docs/en/api/24-vikingbot.md+++ b/docs/en/api/24-vikingbot.md@@ -32,11 +32,13 @@ curl http://localhost:1933/bot/v1/health### chat()-Send a message and wait for the complete reply. Omit `session_id` to create a new session.+Send text and/or images and wait for the complete reply. Omit `session_id` to create a new+session.| Field | Type | Required | Default | Description ||-------|------|----------|---------|-------------|-| `message` | string | Yes | - | Non-empty user message |+| `message` | string | Conditional | `""` | User text; required when `images` is empty |+| `images` | array | Conditional | `[]` | Up to four OpenAI-style `image_url` parts; required when `message` is empty || `session_id` | string | No | Generated | Existing session to continue || `context` | array | No | `null` | Additional messages containing `role` and `content` || `need_reply` | boolean | No | `true` | Whether the Bot should reply |@@ -52,6 +54,30 @@ curl -X POST http://localhost:1933/bot/v1/chat \-d '{"message":"Summarize my project progress","session_id":"optional-session-id"}'```+Images may use an accessible HTTPS URL or an inline Base64 data URL:++```bash+curl -X POST http://localhost:1933/bot/v1/chat \+ -H "Content-Type: application/json" \+ -H "X-API-Key: your-key" \+ -d '{+ "message": "Describe this image",+ "images": [{+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/photo.png"+ }+ }]+ }'+```++Inline Base64 images support JPEG, PNG, GIF, and WebP. Each decoded inline image is limited to+10 MiB; inline SVG and mismatched MIME signatures are rejected. For HTTPS URLs, the Gateway+validates the URL structure but does not fetch or inspect the remote resource, so remote format+support and related errors are provider-specific. Local filesystem paths are rejected. The+optional `detail` field accepts `auto`, `low`, or `high`; omit it for maximum provider+compatibility.+**CLI**```bashdiff --git a/docs/zh/api/24-vikingbot.md b/docs/zh/api/24-vikingbot.mdindex 8f72a5149..476249606 100644--- a/docs/zh/api/24-vikingbot.md+++ b/docs/zh/api/24-vikingbot.md@@ -32,11 +32,12 @@ curl http://localhost:1933/bot/v1/health### chat()-发送一条消息并等待完整回复。`session_id` 可省略;省略时 Gateway 会创建新会话。+发送文本和/或图片并等待完整回复。`session_id` 可省略;省略时 Gateway 会创建新会话。| 字段 | 类型 | 必填 | 默认值 | 说明 ||------|------|------|--------|------|-| `message` | string | 是 | - | 非空的用户消息 |+| `message` | string | 条件必填 | `""` | 用户文本;`images` 为空时必填 |+| `images` | array | 条件必填 | `[]` | 最多 4 个 OpenAI 风格的 `image_url`;`message` 为空时必填 || `session_id` | string | 否 | 自动生成 | 继续已有会话时传入 || `context` | array | 否 | `null` | 额外上下文消息,每项包含 `role` 和 `content` || `need_reply` | boolean | 否 | `true` | 是否需要 Bot 回复 |@@ -52,6 +53,28 @@ curl -X POST http://localhost:1933/bot/v1/chat \-d '{"message":"总结我的项目进展","session_id":"optional-session-id"}'```+图片可以使用模型可访问的 HTTPS URL,或内联 Base64 Data URL:++```bash+curl -X POST http://localhost:1933/bot/v1/chat \+ -H "Content-Type: application/json" \+ -H "X-API-Key: your-key" \+ -d '{+ "message": "描述这张图片",+ "images": [{+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/photo.png"+ }+ }]+ }'+```++内联 Base64 图片支持 JPEG、PNG、GIF 和 WebP,解码后单张最大 10 MiB;内联 SVG 和 MIME+签名不匹配的图片会被拒绝。对于 HTTPS URL,Gateway 只校验 URL 结构,不会下载或检查远程+资源,因此远程格式支持及相关错误由具体 provider 决定。本地文件路径仍会被拒绝。可选的+`detail` 支持 `auto`、`low`、`high`;为获得最好的模型兼容性,建议省略。+**CLI**```bashdiff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.pyindex 05da60d8a..cbd7c56e0 100644--- a/openviking/models/vlm/backends/litellm_vlm.py+++ b/openviking/models/vlm/backends/litellm_vlm.py@@ -17,6 +17,7 @@ from litellm import acompletion, completionfrom openviking.telemetry import tracerfrom openviking.utils.model_retry import retry_async, retry_sync+from openviking.utils.multimodal import redact_image_data_urlsfrom openviking_cli.utils import get_loggerfrom ..base import ToolCall, VLMBase, VLMResponse@@ -449,7 +450,9 @@ class LiteLLMVLMProvider(VLMBase):"""Get text completion asynchronously."""kwargs = self._build_text_kwargs(prompt, thinking, tools, tool_choice, messages)# 用 tracer.info 打印请求- tracer.info(f"request: {json.dumps(kwargs, ensure_ascii=False, indent=2)}")+ tracer.info(+ f"request: {json.dumps(redact_image_data_urls(kwargs), ensure_ascii=False, indent=2)}"+ )async def _call() -> Union[str, VLMResponse]:t0 = time.perf_counter()diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.pyindex 7c629209f..c2a658a25 100644--- a/openviking/models/vlm/backends/openai_vlm.py+++ b/openviking/models/vlm/backends/openai_vlm.py@@ -11,6 +11,7 @@ from urllib.parse import urlparsefrom openviking.telemetry import tracerfrom openviking.utils.async_client_cache import LoopScopedAsyncClientCache+from openviking.utils.multimodal import redact_image_data_urlsfrom openviking_cli.utils import get_loggertry:@@ -344,7 +345,9 @@ class OpenAIVLM(VLMBase):return await self._extract_completion_content_async(response, elapsed)# 用 tracer.info 打印请求- tracer.info(f"messages={json.dumps(kwargs, ensure_ascii=False, indent=2)}")+ tracer.info(+ f"messages={json.dumps(redact_image_data_urls(kwargs), ensure_ascii=False, indent=2)}"+ )return await retry_async(_call,diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.pyindex 485b50b06..0795b02ef 100644--- a/openviking/models/vlm/backends/volcengine_vlm.py+++ b/openviking/models/vlm/backends/volcengine_vlm.py@@ -10,6 +10,7 @@ from pathlib import Pathfrom typing import Any, Dict, List, Optional, Unionfrom openviking.telemetry import tracer+from openviking.utils.multimodal import redact_image_data_urlsfrom openviking_cli.utils import get_loggerfrom ..base import ToolCall, VLMResponse@@ -149,7 +150,7 @@ class VolcEngineVLM(OpenAIVLM):return resultreturn self._clean_response(str(result))- @tracer("volcengine.vlm.call", ignore_result=True, ignore_args=False)+ @tracer("volcengine.vlm.call", ignore_result=True, ignore_args=["messages"])async def get_completion_async(self,prompt: str = "",@@ -175,7 +176,10 @@ class VolcEngineVLM(OpenAIVLM):kwargs["tool_choice"] = tool_choice or "auto"# 用 tracer.info 打印请求- tracer.info(f"request: {json.dumps(kwargs_messages, ensure_ascii=False, indent=2)}")+ tracer.info(+ "request: "+ f"{json.dumps(redact_image_data_urls(kwargs_messages), ensure_ascii=False, indent=2)}"+ )if tools:tracer.info(f"tools: {json.dumps([t['function']['name'] for t in tools], ensure_ascii=False)}"diff --git a/openviking/utils/multimodal.py b/openviking/utils/multimodal.pynew file mode 100644index 000000000..997681ddc--- /dev/null+++ b/openviking/utils/multimodal.py@@ -0,0 +1,23 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Helpers for handling multimodal request data safely."""++from __future__ import annotations++from typing import Any+++def redact_image_data_urls(value: Any) -> Any:+ """Return a copy with inline image payloads replaced by compact placeholders."""+ if isinstance(value, str):+ if value.lower().startswith("data:image/") and ";base64," in value.lower():+ header, payload = value.split(",", 1)+ return f"{header},[redacted {len(payload)} base64 chars]"+ return value+ if isinstance(value, list):+ return [redact_image_data_urls(item) for item in value]+ if isinstance(value, tuple):+ return tuple(redact_image_data_urls(item) for item in value)+ if isinstance(value, dict):+ return {key: redact_image_data_urls(item) for key, item in value.items()}+ return valuediff --git a/tests/unit/test_vikingbot_chat_images.py b/tests/unit/test_vikingbot_chat_images.pynew file mode 100644index 000000000..2cf5f1071--- /dev/null+++ b/tests/unit/test_vikingbot_chat_images.py@@ -0,0 +1,242 @@+import base64++import httpx+import pytest+from fastapi import FastAPI+from pydantic import ValidationError+from vikingbot.agent.context import ContextBuilder+from vikingbot.bus.queue import MessageBus+from vikingbot.channels.openapi import OpenAPIChannel, OpenAPIChannelConfig+from vikingbot.channels.openapi_models import ChatRequest++from openviking.utils.multimodal import redact_image_data_urls+++def _data_url(mime_type: str, data: bytes) -> str:+ return f"data:{mime_type};base64,{base64.b64encode(data).decode()}"+++PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"+++def test_chat_request_accepts_text_only():+ request = ChatRequest(message="hello")++ assert request.message == "hello"+ assert request.images == []+++def test_chat_request_accepts_https_and_base64_images_without_text():+ inline_url = _data_url("image/png", PNG_SIGNATURE)++ request = ChatRequest(+ images=[+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png", "detail": "high"},+ },+ {+ "type": "image_url",+ "image_url": {"url": inline_url},+ },+ ]+ )++ assert request.message == ""+ assert request.images[0].image_url.detail == "high"+ assert request.images[1].image_url.url == inline_url+++def test_chat_request_passes_https_url_without_fetching_remote_content():+ remote_url = "https://example.com/image.svg?version=1"++ request = ChatRequest(+ images=[+ {+ "type": "image_url",+ "image_url": {"url": remote_url},+ }+ ]+ )++ assert request.images[0].image_url.url == remote_url+++ "url",+ [+ "http://example.com/image.png",+ "file:///tmp/image.png",+ "data:image/svg+xml;base64,PHN2Zz4=",+ _data_url("image/jpeg", PNG_SIGNATURE),+ "data:image/png;base64,not-valid-base64!",+ ],+)+def test_chat_request_rejects_unsafe_or_invalid_image_urls(url):+ with pytest.raises(ValidationError):+ ChatRequest(images=[{"type": "image_url", "image_url": {"url": url}}])+++def test_chat_request_requires_text_or_image():+ with pytest.raises(ValidationError, match="message or at least one image is required"):+ ChatRequest()+++def test_chat_request_limits_image_count():+ images = [+ {+ "type": "image_url",+ "image_url": {"url": f"https://example.com/image-{index}.png"},+ }+ for index in range(5)+ ]++ with pytest.raises(ValidationError):+ ChatRequest(images=images)+++async def test_chat_route_does_not_reflect_invalid_base64_input(tmp_path):+ secret = "sensitive-image-content-" * 10_000+ data_url = _data_url("image/png", secret.encode())+ app = FastAPI()+ channel = OpenAPIChannel(+ config=OpenAPIChannelConfig(),+ bus=MessageBus(),+ workspace_path=tmp_path,+ app=app,+ )+ channel._setup_routes()++ transport = httpx.ASGITransport(app=app)+ async with httpx.AsyncClient(transport=transport, base_url="http://127.0.0.1") as client:+ response = await client.post(+ "/bot/v1/chat",+ json={+ "images": [+ {+ "type": "image_url",+ "image_url": {"url": data_url},+ }+ ]+ },+ )++ assert response.status_code == 422+ assert data_url not in response.text+ assert base64.b64encode(secret.encode()).decode() not in response.text+ assert len(response.content) < 2_000+ assert response.json() == {+ "detail": [+ {+ "type": "value_error",+ "loc": ["body", "images", 0, "image_url", "url"],+ "msg": "Value error, image data does not have a supported image signature",+ }+ ]+ }+++def test_openapi_request_media_preserves_openai_image_parts():+ request = ChatRequest(+ message="describe",+ images=[+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png", "detail": "low"},+ }+ ],+ )+ channel = OpenAPIChannel.__new__(OpenAPIChannel)++ assert channel._request_media(request) == [+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png", "detail": "low"},+ }+ ]+++def test_openapi_request_media_omits_unspecified_detail():+ request = ChatRequest(+ images=[+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png"},+ }+ ]+ )+ channel = OpenAPIChannel.__new__(OpenAPIChannel)++ assert channel._request_media(request) == [+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png"},+ }+ ]+++def test_openapi_request_content_uses_image_placeholder_for_image_only_request():+ request = ChatRequest(+ images=[+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png"},+ }+ ]+ )+ channel = OpenAPIChannel.__new__(OpenAPIChannel)++ assert channel._request_content(request) == "[Image]"+++def test_context_builder_passes_remote_and_inline_images_to_model():+ inline_url = _data_url("image/png", PNG_SIGNATURE)+ builder = ContextBuilder.__new__(ContextBuilder)++ content = builder._build_user_content(+ "compare",+ [+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png", "detail": "high"},+ },+ inline_url,+ ],+ )++ assert content == [+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png", "detail": "high"},+ },+ {"type": "image_url", "image_url": {"url": inline_url}},+ {"type": "text", "text": "compare"},+ ]+++def test_context_builder_keeps_trusted_local_file_support(tmp_path):+ image_path = tmp_path / "image.png"+ image_path.write_bytes(PNG_SIGNATURE)+ builder = ContextBuilder.__new__(ContextBuilder)++ content = builder._build_user_content("describe", [str(image_path)])++ assert isinstance(content, list)+ assert content[0]["image_url"]["url"] == _data_url("image/png", PNG_SIGNATURE)+ assert content[-1] == {"type": "text", "text": "describe"}+++def test_redact_image_data_urls_does_not_mutate_request_messages():+ inline_url = _data_url("image/png", PNG_SIGNATURE)+ messages = [+ {+ "role": "user",+ "content": [{"type": "image_url", "image_url": {"url": inline_url}}],+ }+ ]++ redacted = redact_image_data_urls(messages)++ assert "redacted" in redacted[0]["content"][0]["image_url"]["url"]+ assert messages[0]["content"][0]["image_url"]["url"] == inline_urldiff --git a/web-studio/types/ov-server/bot/v1/chat.ts b/web-studio/types/ov-server/bot/v1/chat.tsindex f40b78a2b..8ac81bd5c 100644--- a/web-studio/types/ov-server/bot/v1/chat.ts+++ b/web-studio/types/ov-server/bot/v1/chat.ts@@ -7,9 +7,18 @@ export type ChatStreamEventType =| 'content_delta'| 'reasoning_delta'+export type BotChatImage = {+ type: 'image_url'+ image_url: {+ url: string+ detail?: 'auto' | 'low' | 'high'+ }+}+export type BotChatRequest = {channel_id?: string- message: string+ images?: BotChatImage[]+ message?: stringneed_reply?: booleansession_id?: stringstream?: boolean@@ -23,7 +32,9 @@ export type BotChatResponse = {timestamp: string}-export type BotChatStreamEvent<TEvent extends ChatStreamEventType = ChatStreamEventType> = {+export type BotChatStreamEvent<+ TEvent extends ChatStreamEventType = ChatStreamEventType,+> = {data: unknownevent: TEventtimestamp: string