Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(bot): support image inputs in VikingBot Chat API (#3619)

DuTao6b416cd66d54
diff --git a/bot/vikingbot/agent/context.py b/bot/vikingbot/agent/context.py
index 854537b83..ac047d2a8 100644
--- a/bot/vikingbot/agent/context.py
+++ b/bot/vikingbot/agent/context.py
@@ -347,7 +347,7 @@ IMPORTANT:
self,
history: list[dict[str, Any]],
current_message: str,
- media: list[str] | None = None,
+ media: list[str | dict[str, Any]] | None = None,
session_key: SessionKey | None = None,
ov_tools_enable: bool = True,
profile_user_list: list[str] | None = None,
@@ -442,13 +442,41 @@ IMPORTANT:
return messages
- def _build_user_content(self, text: str, media: list[str] | None) -> str | list[dict[str, Any]]:
- """Build user message content with optional base64-encoded images."""
+ def _build_user_content(
+ self,
+ text: str,
+ media: list[str | dict[str, Any]] | None,
+ ) -> str | list[dict[str, Any]]:
+ """Build user content from trusted local paths or validated image URL parts."""
if not media:
return text
images = []
- for path in media:
+ for item in media:
+ if isinstance(item, dict):
+ image_url = item.get("image_url")
+ url = image_url.get("url") if isinstance(image_url, dict) else None
+ if item.get("type") == "image_url" and isinstance(url, str):
+ images.append(
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": url,
+ **(
+ {"detail": image_url["detail"]}
+ if image_url.get("detail")
+ else {}
+ ),
+ },
+ }
+ )
+ continue
+
+ if item.startswith(("https://", "data:image/")):
+ images.append({"type": "image_url", "image_url": {"url": item}})
+ continue
+
+ path = item
p = Path(path)
mime, _ = mimetypes.guess_type(path)
if not p.is_file() or not mime or not mime.startswith("image/"):
diff --git a/bot/vikingbot/bus/events.py b/bot/vikingbot/bus/events.py
index 02da108b2..9492634d5 100644
--- a/bot/vikingbot/bus/events.py
+++ b/bot/vikingbot/bus/events.py
@@ -37,7 +37,7 @@ class InboundMessage:
timestamp: datetime = field(default_factory=datetime.now)
sender_name: str | None = None
need_reply: bool = True
- media: list[str] = field(default_factory=list) # Media URLs
+ media: list[str | dict[str, Any]] = field(default_factory=list) # Paths or image_url parts
metadata: dict[str, Any] = field(default_factory=dict) # Channel-specific data
openviking_connection: dict[str, Any] | None = None # Internal OpenViking identity
diff --git a/bot/vikingbot/channels/openapi.py b/bot/vikingbot/channels/openapi.py
index a9d1434ef..553a86be7 100644
--- a/bot/vikingbot/channels/openapi.py
+++ b/bot/vikingbot/channels/openapi.py
@@ -13,7 +13,8 @@ from urllib.parse import urlsplit
import httpx
from fastapi import APIRouter, Depends, FastAPI, Header, HTTPException, Request
-from fastapi.responses import Response, StreamingResponse
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse, Response, StreamingResponse
from loguru import logger
from vikingbot.bus.events import InboundMessage, OutboundEventType, OutboundMessage
@@ -66,6 +67,35 @@ HOP_BY_HOP_HEADERS = {
}
+def _public_validation_error(error: Any) -> dict[str, Any]:
+ """Keep validation diagnostics without reflecting request data."""
+ if not isinstance(error, dict):
+ return {
+ "type": "value_error",
+ "loc": ["request"],
+ "msg": "Invalid request value",
+ }
+
+ location = error.get("loc", ("request",))
+ if not isinstance(location, (list, tuple)):
+ location = (location,)
+ return {
+ "type": str(error.get("type") or "value_error"),
+ "loc": [item if isinstance(item, (str, int)) else str(item) for item in location],
+ "msg": str(error.get("msg") or "Invalid request value"),
+ }
+
+
+async def _request_validation_error_handler(
+ _request: Request,
+ exc: RequestValidationError,
+) -> JSONResponse:
+ # Pydantic validation errors include the original `input` by default. Never
+ # return it here: image inputs may contain large or sensitive Base64 data.
+ errors = [_public_validation_error(error) for error in exc.errors()]
+ return JSONResponse(status_code=422, content={"detail": errors})
+
+
@dataclass(frozen=True)
class GatewayRequestAuth:
"""Resolved gateway-level auth facts for one HTTP request."""
@@ -998,9 +1028,7 @@ class OpenAPIChannel(BaseChannel):
detail=OPENVIKING_UPSTREAM_NOT_CONFIGURED_DETAIL,
)
await self._assert_runtime_upstream_auth_mode(
- self._identity_headers_from_connection(
- compile_request.openviking_connection
- )
+ self._identity_headers_from_connection(compile_request.openviking_connection)
)
compile_request._principal_scope = self._connection_principal_scope(
compile_request.openviking_connection
@@ -1200,6 +1228,11 @@ class OpenAPIChannel(BaseChannel):
logger.warning("No external FastAPI app provided, cannot setup routes")
return
+ self._app.add_exception_handler(
+ RequestValidationError,
+ _request_validation_error_handler,
+ )
+
# Get the router and include it at root path
# Note: openviking-server adds its own /bot/v1 prefix when proxying
router = self.get_router()
@@ -1220,6 +1253,12 @@ class OpenAPIChannel(BaseChannel):
disabled_tools.append("message")
return {"disabled_tools": disabled_tools}
+ def _request_media(self, request: ChatRequest) -> list[dict[str, Any]]:
+ return [image.model_dump(mode="json", exclude_none=True) for image in request.images]
+
+ def _request_content(self, request: ChatRequest) -> str:
+ return request.message or "[Image]"
+
def _request_openviking_connection(self, request: ChatRequest) -> dict[str, Any] | None:
if request.openviking_connection:
connection = request.openviking_connection.model_dump(exclude_none=True)
@@ -1266,7 +1305,7 @@ class OpenAPIChannel(BaseChannel):
)
# Build content with context if provided
- content = request.message
+ content = self._request_content(request)
if request.context:
# Context is handled separately by session manager
pass
@@ -1277,6 +1316,7 @@ class OpenAPIChannel(BaseChannel):
sender_id=user_id,
actor_peer_id=self._request_actor_peer_id(request, user_id),
content=content,
+ media=self._request_media(request),
metadata=self._request_metadata(request),
openviking_connection=self._request_openviking_connection(request),
)
@@ -1347,7 +1387,8 @@ class OpenAPIChannel(BaseChannel):
session_key=session_key,
sender_id=user_id,
actor_peer_id=self._request_actor_peer_id(request, user_id),
- content=request.message,
+ content=self._request_content(request),
+ media=self._request_media(request),
metadata=self._request_metadata(request),
openviking_connection=self._request_openviking_connection(request),
)
@@ -1421,7 +1462,7 @@ class OpenAPIChannel(BaseChannel):
)
# Build content with context if provided
- content = request.message
+ content = self._request_content(request)
if request.context:
# Context is handled separately by session manager
pass
@@ -1433,6 +1474,7 @@ class OpenAPIChannel(BaseChannel):
actor_peer_id=self._request_actor_peer_id(request, user_id),
content=content,
need_reply=request.need_reply,
+ media=self._request_media(request),
metadata=self._request_metadata(request),
openviking_connection=self._request_openviking_connection(request),
)
@@ -1510,7 +1552,8 @@ class OpenAPIChannel(BaseChannel):
session_key=session_key,
sender_id=user_id,
actor_peer_id=self._request_actor_peer_id(request, user_id),
- content=request.message,
+ content=self._request_content(request),
+ media=self._request_media(request),
metadata=self._request_metadata(request),
openviking_connection=self._request_openviking_connection(request),
)
diff --git a/bot/vikingbot/channels/openapi_models.py b/bot/vikingbot/channels/openapi_models.py
index 4abd9b34b..1ac16855a 100644
--- a/bot/vikingbot/channels/openapi_models.py
+++ b/bot/vikingbot/channels/openapi_models.py
@@ -1,10 +1,78 @@
"""Pydantic models for OpenAPI channel."""
+import base64
+import binascii
from datetime import datetime
from enum import Enum
-from typing import Any, Dict, List, Optional
-
-from pydantic import BaseModel, Field, PrivateAttr, model_validator
+from typing import Any, Dict, List, Literal, Optional
+from urllib.parse import urlsplit
+
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator, model_validator
+
+MAX_CHAT_IMAGES = 4
+MAX_CHAT_INLINE_IMAGE_BYTES = 10 * 1024 * 1024
+SUPPORTED_CHAT_INLINE_IMAGE_MIME_TYPES = {
+ "image/gif",
+ "image/jpeg",
+ "image/png",
+ "image/webp",
+}
+
+
+def _detect_chat_image_mime(data: bytes) -> str | None:
+ if data.startswith(b"\x89PNG\r\n\x1a\n"):
+ return "image/png"
+ if data.startswith(b"\xff\xd8\xff"):
+ return "image/jpeg"
+ if data.startswith((b"GIF87a", b"GIF89a")):
+ return "image/gif"
+ if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
+ return "image/webp"
+ return None
+
+
+def _validate_chat_image_url(value: str) -> str:
+ if value.startswith("https://"):
+ parsed = urlsplit(value)
+ if not parsed.hostname or parsed.username or parsed.password:
+ raise ValueError("image URL must be an absolute HTTPS URL without credentials")
+ # Match multimodal model APIs: pass remote URLs through without fetching
+ # them. Content-type and format support are therefore provider-specific.
+ return value
+
+ if not value.startswith("data:image/"):
+ raise ValueError("image URL must use HTTPS or an image Base64 data URL")
+
+ try:
+ header, encoded = value.split(",", 1)
+ media_type, encoding = header[5:].split(";", 1)
+ except ValueError as exc:
+ raise ValueError("invalid image data URL") from exc
+
+ media_type = media_type.lower()
+ if media_type not in SUPPORTED_CHAT_INLINE_IMAGE_MIME_TYPES:
+ raise ValueError("unsupported image MIME type; expected JPEG, PNG, GIF, or WebP")
+ if encoding.lower() != "base64" or not encoded:
+ raise ValueError("image data URL must contain Base64-encoded data")
+
+ # Reject oversized payloads before decoding to avoid a large temporary allocation.
+ if len(encoded) > ((MAX_CHAT_INLINE_IMAGE_BYTES + 2) // 3) * 4:
+ raise ValueError(f"decoded image exceeds {MAX_CHAT_INLINE_IMAGE_BYTES} bytes")
+ try:
+ decoded = base64.b64decode(encoded, validate=True)
+ except (binascii.Error, ValueError) as exc:
+ raise ValueError("image data URL contains invalid Base64") from exc
+ if len(decoded) > MAX_CHAT_INLINE_IMAGE_BYTES:
+ raise ValueError(f"decoded image exceeds {MAX_CHAT_INLINE_IMAGE_BYTES} bytes")
+
+ detected_type = _detect_chat_image_mime(decoded)
+ if detected_type is None:
+ raise ValueError("image data does not have a supported image signature")
+ if detected_type != media_type:
+ raise ValueError(
+ f"image MIME type {media_type} does not match detected type {detected_type}"
+ )
+ return value
class MessageRole(str, Enum):
@@ -46,6 +114,35 @@ class ChatMessage(BaseModel):
)
+class ChatImageURL(BaseModel):
+ """OpenAI-compatible image URL descriptor."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ url: str = Field(
+ ...,
+ description="HTTPS passthrough URL or validated Base64 image data URL",
+ )
+ detail: Optional[Literal["auto", "low", "high"]] = Field(
+ default=None,
+ description="Optional image detail level; omitted for maximum provider compatibility",
+ )
+
+ @field_validator("url")
+ @classmethod
+ def validate_url(cls, value: str) -> str:
+ return _validate_chat_image_url(value)
+
+
+class ChatImage(BaseModel):
+ """OpenAI-compatible image input part."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ type: Literal["image_url"] = "image_url"
+ image_url: ChatImageURL
+
+
class OpenVikingConnection(BaseModel):
"""OpenViking identity forwarded by the Studio proxy."""
@@ -66,7 +163,12 @@ class OpenVikingConnection(BaseModel):
class ChatRequest(BaseModel):
"""Request body for chat endpoint."""
- message: str = Field(..., description="User message to send", min_length=1)
+ message: str = Field(default="", description="User message to send")
+ images: List[ChatImage] = Field(
+ default_factory=list,
+ max_length=MAX_CHAT_IMAGES,
+ description="HTTPS passthrough URLs or validated Base64 image inputs",
+ )
session_id: Optional[str] = Field(
default=None, description="Session ID (optional, will create new if not provided)"
)
@@ -89,6 +191,12 @@ class ChatRequest(BaseModel):
description="Authenticated OpenViking connection forwarded by the server proxy",
)
+ @model_validator(mode="after")
+ def validate_message_or_images(self) -> "ChatRequest":
+ if not self.message.strip() and not self.images:
+ raise ValueError("message or at least one image is required")
+ return self
+
class ChatResponse(BaseModel):
"""Response from chat endpoint (non-streaming)."""
diff --git a/bot/vikingbot/cli/commands.py b/bot/vikingbot/cli/commands.py
index 29f48ce3c..7c8224375 100644
--- a/bot/vikingbot/cli/commands.py
+++ b/bot/vikingbot/cli/commands.py
@@ -310,8 +310,6 @@ def _make_provider(config, langfuse_client: Any = None):
Bot's own VLM configuration. Otherwise the complete root VLM configuration
is inherited, including its ordered credentials and failover behavior.
"""
- from vikingbot.providers.litellm_provider import LiteLLMProvider
-
p = config.agents
model = p.model if p else None
temperature = p.temperature if p else 0.7
@@ -431,20 +429,10 @@ def _make_provider(config, langfuse_client: Any = None):
langfuse_client=langfuse_client,
)
- # Fallback: legacy LiteLLMProvider (no explicit provider set)
- if not api_key and not model.startswith("bedrock/"):
- console.print("[yellow]Warning: No API key configured.[/yellow]")
- console.print("You can configure providers later in the Console UI.")
-
- return LiteLLMProvider(
- api_key=api_key,
- api_base=api_base,
- default_model=model,
- extra_headers=extra_headers,
- provider_name=provider_name,
- timeout=timeout,
- thinking=thinking,
- langfuse_client=langfuse_client,
+ raise RuntimeError(
+ "No VLM provider configured for VikingBot. Set bot.agents.provider, "
+ "configure bot.agents.credentials, or inherit the root vlm configuration. "
+ "Set provider to 'litellm' to use LiteLLM."
)
diff --git a/bot/vikingbot/providers/litellm_provider.py b/bot/vikingbot/providers/litellm_provider.py
index 9ae711758..35a35e5ed 100644
--- a/bot/vikingbot/providers/litellm_provider.py
+++ b/bot/vikingbot/providers/litellm_provider.py
@@ -7,6 +7,7 @@ import litellm
from litellm import acompletion
from loguru import logger
+from openviking.utils.multimodal import redact_image_data_urls
from vikingbot.integrations.langfuse import LangfuseClient
from vikingbot.providers.base import (
LLMProvider,
@@ -136,9 +137,7 @@ class LiteLLMProvider(LLMProvider):
kwargs["thinking"] = {"type": "enabled"}
return
- if thinking_param == "dashscope_enable_thinking" or model_lower.startswith(
- "dashscope/"
- ):
+ if thinking_param == "dashscope_enable_thinking" or model_lower.startswith("dashscope/"):
extra_body = dict(kwargs.get("extra_body") or {})
extra_body["enable_thinking"] = True
kwargs["extra_body"] = extra_body
@@ -283,7 +282,7 @@ class LiteLLMProvider(LLMProvider):
name="llm-chat",
as_type="generation",
model=model,
- input=messages,
+ input=redact_image_data_urls(messages),
metadata=metadata,
)
if response_id:
diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.py
index de903eada..5ec814c85 100644
--- a/bot/vikingbot/providers/vlm_adapter.py
+++ b/bot/vikingbot/providers/vlm_adapter.py
@@ -13,6 +13,7 @@ from typing import Any
from loguru import logger
from openviking.utils.model_retry import is_retryable_rate_limit_error, rate_limit_retry_delay
+from openviking.utils.multimodal import redact_image_data_urls
from vikingbot.integrations.langfuse import LangfuseClient
from vikingbot.providers.base import (
LLMProvider,
@@ -72,7 +73,7 @@ class VLMProviderAdapter(LLMProvider):
name="llm-chat",
as_type="generation",
model=effective_model,
- input=messages,
+ input=redact_image_data_urls(messages),
metadata=metadata,
)
if response_id:
diff --git a/bot/vikingbot/tests/unit/test_bot_vlm_credentials.py b/bot/vikingbot/tests/unit/test_bot_vlm_credentials.py
index 6e4527fbe..fd1277640 100644
--- a/bot/vikingbot/tests/unit/test_bot_vlm_credentials.py
+++ b/bot/vikingbot/tests/unit/test_bot_vlm_credentials.py
@@ -5,6 +5,7 @@ from unittest.mock import AsyncMock
import pytest
from openviking.models.vlm import MultiCredentialVLM
+from openviking.models.vlm.backends.litellm_vlm import LiteLLMVLMProvider
from vikingbot.config import loader
from vikingbot.config.schema import Config
from vikingbot.providers.vlm_adapter import VLMProviderAdapter
@@ -238,6 +239,50 @@ def test_explicit_bot_model_without_credentials_keeps_single_model_behavior(tmp_
assert provider._vlm.model == "bot-model"
+def test_explicit_litellm_provider_uses_vlm_adapter(tmp_path, monkeypatch):
+ config = _write_config(
+ tmp_path,
+ monkeypatch,
+ {
+ "bot": {
+ "agents": {
+ "provider": "litellm",
+ "model": "openrouter/openai/gpt-4o-mini",
+ "api_key": "bot-key",
+ }
+ }
+ },
+ )
+
+ from vikingbot.cli.commands import _make_provider
+
+ provider = _make_provider(config)
+
+ assert isinstance(provider, VLMProviderAdapter)
+ assert isinstance(provider._vlm, LiteLLMVLMProvider)
+ assert provider._vlm.model == "openrouter/openai/gpt-4o-mini"
+
+
+def test_bot_model_without_provider_rejects_legacy_fallback(tmp_path, monkeypatch):
+ config = _write_config(
+ tmp_path,
+ monkeypatch,
+ {
+ "bot": {
+ "agents": {
+ "model": "openrouter/openai/gpt-4o-mini",
+ "api_key": "bot-key",
+ }
+ }
+ },
+ )
+
+ from vikingbot.cli.commands import _make_provider
+
+ with pytest.raises(RuntimeError, match="Set provider to 'litellm'"):
+ _make_provider(config)
+
+
def test_saving_inherited_config_does_not_turn_root_model_into_bot_override(tmp_path, monkeypatch):
config = _write_config(
tmp_path,
diff --git a/docs/en/api/24-vikingbot.md b/docs/en/api/24-vikingbot.md
index 88b3985d8..3bbe81007 100644
--- a/docs/en/api/24-vikingbot.md
+++ b/docs/en/api/24-vikingbot.md
@@ -32,11 +32,13 @@ curl http://localhost:1933/bot/v1/health
### chat()
-Send a message and wait for the complete reply. Omit `session_id` to create a new session.
+Send text and/or images and wait for the complete reply. Omit `session_id` to create a new
+session.
| Field | Type | Required | Default | Description |
|-------|------|----------|---------|-------------|
-| `message` | string | Yes | - | Non-empty user message |
+| `message` | string | Conditional | `""` | User text; required when `images` is empty |
+| `images` | array | Conditional | `[]` | Up to four OpenAI-style `image_url` parts; required when `message` is empty |
| `session_id` | string | No | Generated | Existing session to continue |
| `context` | array | No | `null` | Additional messages containing `role` and `content` |
| `need_reply` | boolean | No | `true` | Whether the Bot should reply |
@@ -52,6 +54,30 @@ curl -X POST http://localhost:1933/bot/v1/chat \
-d '{"message":"Summarize my project progress","session_id":"optional-session-id"}'
```
+Images may use an accessible HTTPS URL or an inline Base64 data URL:
+
+```bash
+curl -X POST http://localhost:1933/bot/v1/chat \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "message": "Describe this image",
+ "images": [{
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/photo.png"
+ }
+ }]
+ }'
+```
+
+Inline Base64 images support JPEG, PNG, GIF, and WebP. Each decoded inline image is limited to
+10 MiB; inline SVG and mismatched MIME signatures are rejected. For HTTPS URLs, the Gateway
+validates the URL structure but does not fetch or inspect the remote resource, so remote format
+support and related errors are provider-specific. Local filesystem paths are rejected. The
+optional `detail` field accepts `auto`, `low`, or `high`; omit it for maximum provider
+compatibility.
+
**CLI**
```bash
diff --git a/docs/zh/api/24-vikingbot.md b/docs/zh/api/24-vikingbot.md
index 8f72a5149..476249606 100644
--- a/docs/zh/api/24-vikingbot.md
+++ b/docs/zh/api/24-vikingbot.md
@@ -32,11 +32,12 @@ curl http://localhost:1933/bot/v1/health
### chat()
-发送一条消息并等待完整回复。`session_id` 可省略;省略时 Gateway 会创建新会话。
+发送文本和/或图片并等待完整回复。`session_id` 可省略;省略时 Gateway 会创建新会话。
| 字段 | 类型 | 必填 | 默认值 | 说明 |
|------|------|------|--------|------|
-| `message` | string | 是 | - | 非空的用户消息 |
+| `message` | string | 条件必填 | `""` | 用户文本;`images` 为空时必填 |
+| `images` | array | 条件必填 | `[]` | 最多 4 个 OpenAI 风格的 `image_url`;`message` 为空时必填 |
| `session_id` | string | 否 | 自动生成 | 继续已有会话时传入 |
| `context` | array | 否 | `null` | 额外上下文消息,每项包含 `role` 和 `content` |
| `need_reply` | boolean | 否 | `true` | 是否需要 Bot 回复 |
@@ -52,6 +53,28 @@ curl -X POST http://localhost:1933/bot/v1/chat \
-d '{"message":"总结我的项目进展","session_id":"optional-session-id"}'
```
+图片可以使用模型可访问的 HTTPS URL,或内联 Base64 Data URL:
+
+```bash
+curl -X POST http://localhost:1933/bot/v1/chat \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "message": "描述这张图片",
+ "images": [{
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/photo.png"
+ }
+ }]
+ }'
+```
+
+内联 Base64 图片支持 JPEG、PNG、GIF 和 WebP,解码后单张最大 10 MiB;内联 SVG 和 MIME
+签名不匹配的图片会被拒绝。对于 HTTPS URL,Gateway 只校验 URL 结构,不会下载或检查远程
+资源,因此远程格式支持及相关错误由具体 provider 决定。本地文件路径仍会被拒绝。可选的
+`detail` 支持 `auto`、`low`、`high`;为获得最好的模型兼容性,建议省略。
+
**CLI**
```bash
diff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.py
index 05da60d8a..cbd7c56e0 100644
--- a/openviking/models/vlm/backends/litellm_vlm.py
+++ b/openviking/models/vlm/backends/litellm_vlm.py
@@ -17,6 +17,7 @@ from litellm import acompletion, completion
from openviking.telemetry import tracer
from openviking.utils.model_retry import retry_async, retry_sync
+from openviking.utils.multimodal import redact_image_data_urls
from openviking_cli.utils import get_logger
from ..base import ToolCall, VLMBase, VLMResponse
@@ -449,7 +450,9 @@ class LiteLLMVLMProvider(VLMBase):
"""Get text completion asynchronously."""
kwargs = self._build_text_kwargs(prompt, thinking, tools, tool_choice, messages)
# 用 tracer.info 打印请求
- tracer.info(f"request: {json.dumps(kwargs, ensure_ascii=False, indent=2)}")
+ tracer.info(
+ f"request: {json.dumps(redact_image_data_urls(kwargs), ensure_ascii=False, indent=2)}"
+ )
async def _call() -> Union[str, VLMResponse]:
t0 = time.perf_counter()
diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.py
index 7c629209f..c2a658a25 100644
--- a/openviking/models/vlm/backends/openai_vlm.py
+++ b/openviking/models/vlm/backends/openai_vlm.py
@@ -11,6 +11,7 @@ from urllib.parse import urlparse
from openviking.telemetry import tracer
from openviking.utils.async_client_cache import LoopScopedAsyncClientCache
+from openviking.utils.multimodal import redact_image_data_urls
from openviking_cli.utils import get_logger
try:
@@ -344,7 +345,9 @@ class OpenAIVLM(VLMBase):
return await self._extract_completion_content_async(response, elapsed)
# 用 tracer.info 打印请求
- tracer.info(f"messages={json.dumps(kwargs, ensure_ascii=False, indent=2)}")
+ tracer.info(
+ f"messages={json.dumps(redact_image_data_urls(kwargs), ensure_ascii=False, indent=2)}"
+ )
return await retry_async(
_call,
diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.py
index 485b50b06..0795b02ef 100644
--- a/openviking/models/vlm/backends/volcengine_vlm.py
+++ b/openviking/models/vlm/backends/volcengine_vlm.py
@@ -10,6 +10,7 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from openviking.telemetry import tracer
+from openviking.utils.multimodal import redact_image_data_urls
from openviking_cli.utils import get_logger
from ..base import ToolCall, VLMResponse
@@ -149,7 +150,7 @@ class VolcEngineVLM(OpenAIVLM):
return result
return self._clean_response(str(result))
- @tracer("volcengine.vlm.call", ignore_result=True, ignore_args=False)
+ @tracer("volcengine.vlm.call", ignore_result=True, ignore_args=["messages"])
async def get_completion_async(
self,
prompt: str = "",
@@ -175,7 +176,10 @@ class VolcEngineVLM(OpenAIVLM):
kwargs["tool_choice"] = tool_choice or "auto"
# 用 tracer.info 打印请求
- tracer.info(f"request: {json.dumps(kwargs_messages, ensure_ascii=False, indent=2)}")
+ tracer.info(
+ "request: "
+ f"{json.dumps(redact_image_data_urls(kwargs_messages), ensure_ascii=False, indent=2)}"
+ )
if tools:
tracer.info(
f"tools: {json.dumps([t['function']['name'] for t in tools], ensure_ascii=False)}"
diff --git a/openviking/utils/multimodal.py b/openviking/utils/multimodal.py
new file mode 100644
index 000000000..997681ddc
--- /dev/null
+++ b/openviking/utils/multimodal.py
@@ -0,0 +1,23 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Helpers for handling multimodal request data safely."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def redact_image_data_urls(value: Any) -> Any:
+ """Return a copy with inline image payloads replaced by compact placeholders."""
+ if isinstance(value, str):
+ if value.lower().startswith("data:image/") and ";base64," in value.lower():
+ header, payload = value.split(",", 1)
+ return f"{header},[redacted {len(payload)} base64 chars]"
+ return value
+ if isinstance(value, list):
+ return [redact_image_data_urls(item) for item in value]
+ if isinstance(value, tuple):
+ return tuple(redact_image_data_urls(item) for item in value)
+ if isinstance(value, dict):
+ return {key: redact_image_data_urls(item) for key, item in value.items()}
+ return value
diff --git a/tests/unit/test_vikingbot_chat_images.py b/tests/unit/test_vikingbot_chat_images.py
new file mode 100644
index 000000000..2cf5f1071
--- /dev/null
+++ b/tests/unit/test_vikingbot_chat_images.py
@@ -0,0 +1,242 @@
+import base64
+
+import httpx
+import pytest
+from fastapi import FastAPI
+from pydantic import ValidationError
+from vikingbot.agent.context import ContextBuilder
+from vikingbot.bus.queue import MessageBus
+from vikingbot.channels.openapi import OpenAPIChannel, OpenAPIChannelConfig
+from vikingbot.channels.openapi_models import ChatRequest
+
+from openviking.utils.multimodal import redact_image_data_urls
+
+
+def _data_url(mime_type: str, data: bytes) -> str:
+ return f"data:{mime_type};base64,{base64.b64encode(data).decode()}"
+
+
+PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
+
+
+def test_chat_request_accepts_text_only():
+ request = ChatRequest(message="hello")
+
+ assert request.message == "hello"
+ assert request.images == []
+
+
+def test_chat_request_accepts_https_and_base64_images_without_text():
+ inline_url = _data_url("image/png", PNG_SIGNATURE)
+
+ request = ChatRequest(
+ images=[
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png", "detail": "high"},
+ },
+ {
+ "type": "image_url",
+ "image_url": {"url": inline_url},
+ },
+ ]
+ )
+
+ assert request.message == ""
+ assert request.images[0].image_url.detail == "high"
+ assert request.images[1].image_url.url == inline_url
+
+
+def test_chat_request_passes_https_url_without_fetching_remote_content():
+ remote_url = "https://example.com/image.svg?version=1"
+
+ request = ChatRequest(
+ images=[
+ {
+ "type": "image_url",
+ "image_url": {"url": remote_url},
+ }
+ ]
+ )
+
+ assert request.images[0].image_url.url == remote_url
+
+
+ "url",
+ [
+ "http://example.com/image.png",
+ "file:///tmp/image.png",
+ "data:image/svg+xml;base64,PHN2Zz4=",
+ _data_url("image/jpeg", PNG_SIGNATURE),
+ "data:image/png;base64,not-valid-base64!",
+ ],
+)
+def test_chat_request_rejects_unsafe_or_invalid_image_urls(url):
+ with pytest.raises(ValidationError):
+ ChatRequest(images=[{"type": "image_url", "image_url": {"url": url}}])
+
+
+def test_chat_request_requires_text_or_image():
+ with pytest.raises(ValidationError, match="message or at least one image is required"):
+ ChatRequest()
+
+
+def test_chat_request_limits_image_count():
+ images = [
+ {
+ "type": "image_url",
+ "image_url": {"url": f"https://example.com/image-{index}.png"},
+ }
+ for index in range(5)
+ ]
+
+ with pytest.raises(ValidationError):
+ ChatRequest(images=images)
+
+
+async def test_chat_route_does_not_reflect_invalid_base64_input(tmp_path):
+ secret = "sensitive-image-content-" * 10_000
+ data_url = _data_url("image/png", secret.encode())
+ app = FastAPI()
+ channel = OpenAPIChannel(
+ config=OpenAPIChannelConfig(),
+ bus=MessageBus(),
+ workspace_path=tmp_path,
+ app=app,
+ )
+ channel._setup_routes()
+
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://127.0.0.1") as client:
+ response = await client.post(
+ "/bot/v1/chat",
+ json={
+ "images": [
+ {
+ "type": "image_url",
+ "image_url": {"url": data_url},
+ }
+ ]
+ },
+ )
+
+ assert response.status_code == 422
+ assert data_url not in response.text
+ assert base64.b64encode(secret.encode()).decode() not in response.text
+ assert len(response.content) < 2_000
+ assert response.json() == {
+ "detail": [
+ {
+ "type": "value_error",
+ "loc": ["body", "images", 0, "image_url", "url"],
+ "msg": "Value error, image data does not have a supported image signature",
+ }
+ ]
+ }
+
+
+def test_openapi_request_media_preserves_openai_image_parts():
+ request = ChatRequest(
+ message="describe",
+ images=[
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png", "detail": "low"},
+ }
+ ],
+ )
+ channel = OpenAPIChannel.__new__(OpenAPIChannel)
+
+ assert channel._request_media(request) == [
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png", "detail": "low"},
+ }
+ ]
+
+
+def test_openapi_request_media_omits_unspecified_detail():
+ request = ChatRequest(
+ images=[
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png"},
+ }
+ ]
+ )
+ channel = OpenAPIChannel.__new__(OpenAPIChannel)
+
+ assert channel._request_media(request) == [
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png"},
+ }
+ ]
+
+
+def test_openapi_request_content_uses_image_placeholder_for_image_only_request():
+ request = ChatRequest(
+ images=[
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png"},
+ }
+ ]
+ )
+ channel = OpenAPIChannel.__new__(OpenAPIChannel)
+
+ assert channel._request_content(request) == "[Image]"
+
+
+def test_context_builder_passes_remote_and_inline_images_to_model():
+ inline_url = _data_url("image/png", PNG_SIGNATURE)
+ builder = ContextBuilder.__new__(ContextBuilder)
+
+ content = builder._build_user_content(
+ "compare",
+ [
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png", "detail": "high"},
+ },
+ inline_url,
+ ],
+ )
+
+ assert content == [
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png", "detail": "high"},
+ },
+ {"type": "image_url", "image_url": {"url": inline_url}},
+ {"type": "text", "text": "compare"},
+ ]
+
+
+def test_context_builder_keeps_trusted_local_file_support(tmp_path):
+ image_path = tmp_path / "image.png"
+ image_path.write_bytes(PNG_SIGNATURE)
+ builder = ContextBuilder.__new__(ContextBuilder)
+
+ content = builder._build_user_content("describe", [str(image_path)])
+
+ assert isinstance(content, list)
+ assert content[0]["image_url"]["url"] == _data_url("image/png", PNG_SIGNATURE)
+ assert content[-1] == {"type": "text", "text": "describe"}
+
+
+def test_redact_image_data_urls_does_not_mutate_request_messages():
+ inline_url = _data_url("image/png", PNG_SIGNATURE)
+ messages = [
+ {
+ "role": "user",
+ "content": [{"type": "image_url", "image_url": {"url": inline_url}}],
+ }
+ ]
+
+ redacted = redact_image_data_urls(messages)
+
+ assert "redacted" in redacted[0]["content"][0]["image_url"]["url"]
+ assert messages[0]["content"][0]["image_url"]["url"] == inline_url
diff --git a/web-studio/types/ov-server/bot/v1/chat.ts b/web-studio/types/ov-server/bot/v1/chat.ts
index f40b78a2b..8ac81bd5c 100644
--- a/web-studio/types/ov-server/bot/v1/chat.ts
+++ b/web-studio/types/ov-server/bot/v1/chat.ts
@@ -7,9 +7,18 @@ export type ChatStreamEventType =
| 'content_delta'
| 'reasoning_delta'
+export type BotChatImage = {
+ type: 'image_url'
+ image_url: {
+ url: string
+ detail?: 'auto' | 'low' | 'high'
+ }
+}
+
export type BotChatRequest = {
channel_id?: string
- message: string
+ images?: BotChatImage[]
+ message?: string
need_reply?: boolean
session_id?: string
stream?: boolean
@@ -23,7 +32,9 @@ export type BotChatResponse = {
timestamp: string
}
-export type BotChatStreamEvent<TEvent extends ChatStreamEventType = ChatStreamEventType> = {
+export type BotChatStreamEvent<
+ TEvent extends ChatStreamEventType = ChatStreamEventType,
+> = {
data: unknown
event: TEvent
timestamp: string