Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(session): Support image message extraction (#2578)
agenta4aefac1f793
diff --git a/benchmark/locomo/openviking/README.md b/benchmark/locomo/openviking/README.mdindex 0e15a6e07..3352636f2 100644--- a/benchmark/locomo/openviking/README.md+++ b/benchmark/locomo/openviking/README.md@@ -24,6 +24,13 @@ LoCoMo samples are imported under `viking://user/sample_{idx}/memories`.Evaluation uses the same `sample_{idx}` user id, so import and eval must use thesame dataset order.+LoCoMo image evidence is imported as text from `blip_caption` by default. This+keeps benchmark runs compatible with older OpenViking servers and avoids making+results depend on whether the configured VLM can download external image URLs.+Use `--use-image-url` when you explicitly want to import structured+`image_url` parts and let OpenViking's VLM describe images during memory+extraction.+## Import```bash@@ -35,6 +42,8 @@ python benchmark/locomo/openviking/import_to_ov.py \```Use `--force-ingest` only when intentionally re-importing existing samples.+Use `--use-image-url` only for multimodal import experiments; the default uses+LoCoMo's `blip_caption` text.## Smoke Testdiff --git a/benchmark/locomo/openviking/import_to_ov.py b/benchmark/locomo/openviking/import_to_ov.pyindex 20752ab2e..178776d3c 100644--- a/benchmark/locomo/openviking/import_to_ov.py+++ b/benchmark/locomo/openviking/import_to_ov.py@@ -171,19 +171,30 @@ def _normalize_image_urls(value: Any) -> List[str]:return [str(value)]-def build_openviking_message_parts(msg: Dict[str, Any]) -> List[Dict[str, Any]]:+def build_openviking_message_parts(+ msg: Dict[str, Any],+ *,+ use_image_url: bool = False,+) -> List[Dict[str, Any]]:"""Build OpenViking message parts from a LoCoMo message."""parts: List[Dict[str, Any]] = [{"type": "text", "text": msg["text"]}]caption = str(msg.get("blip_caption", "") or "")for image_url in _normalize_image_urls(msg.get("img_url")):- image_part: Dict[str, Any] = {- "type": "image_url",- "image_url": {"url": image_url},- }- if caption:- image_part["caption"] = caption- parts.append(image_part)+ if use_image_url:+ parts.append(+ {+ "type": "image_url",+ "image_url": {"url": image_url},+ }+ )+ else:+ parts.append(+ {+ "type": "text",+ "text": f"[Image URL: {image_url}; BLIP caption: {caption}]",+ }+ )return parts@@ -342,6 +353,7 @@ async def viking_ingest(session_time: Optional[str] = None,user_id: Optional[str] = None,agent_id: Optional[str] = None,+ use_image_url: bool = False,) -> Dict[str, int]:"""Save messages to OpenViking via OpenViking SDK client.Returns token usage dict with embedding and vlm token counts.@@ -384,7 +396,10 @@ async def viking_ingest(await client.add_message(session_id=session_id,role=msg["role"],- parts=build_openviking_message_parts(msg),+ parts=build_openviking_message_parts(+ msg,+ use_image_url=use_image_url,+ ),created_at=msg_created_at,)@@ -455,6 +470,7 @@ async def process_single_session(meta.get("date_time"),user_id=str(sample_id),agent_id=str(sample_id),+ use_image_url=args.use_image_url,)chunk_usage = result["token_usage"]for key in token_usage:@@ -787,6 +803,12 @@ def main():default=False,help="Clear all existing ingest records before running",)+ parser.add_argument(+ "--use-image-url",+ action="store_true",+ default=False,+ help="Import LoCoMo images as structured image_url parts. Default: false; use blip_caption text for compatibility.",+ )args = parser.parse_args()# 确保输出目录存在diff --git a/docs/en/api/05-sessions.md b/docs/en/api/05-sessions.mdindex 0a61588ac..e2057cea7 100644--- a/docs/en/api/05-sessions.md+++ b/docs/en/api/05-sessions.md@@ -556,10 +556,11 @@ ov session delete a1b2c3d4#### 1. API Implementation Introduction-Add a message to the session. Supports two modes: simple text mode and Parts mode (supporting text, context references, tool calls, etc.).+Add a message to the session. Supports two modes: simple text mode and Parts mode (supporting text, image URLs, context references, tool calls, etc.).**Part Types:**- `TextPart`: Pure text content+- `ImagePart`: OpenAI-style image URL content. During memory extraction, OpenViking can use the configured VLM to turn images into text descriptions.- `ContextPart`: Context reference pointing to resources or memories- `ToolPart`: Tool call and result@@ -591,11 +592,14 @@ Add a message to the session. Supports two modes: simple text mode and Parts mod**Part Types (Python SDK)**```python-from openviking.message import TextPart, ContextPart, ToolPart+from openviking.message import TextPart, ImagePart, ContextPart, ToolPart# Text contentTextPart(text="Hello, how can I help?")+# Image URL content+ImagePart(url="https://example.com/photo.png", detail="auto")+# Context referenceContextPart(uri="viking://resources/docs/auth/",@@ -661,13 +665,25 @@ curl -X POST http://localhost:1933/api/v1/sessions/a1b2c3d4/messages \{"type": "tool", "tool_id": "call_123", "tool_name": "search_web", "tool_input": {"query": "OAuth"}, "tool_status": "completed", "tool_output": "Results..."}]}'++# Add user message with an image URL+curl -X POST http://localhost:1933/api/v1/sessions/a1b2c3d4/messages \+ -H "Content-Type: application/json" \+ -H "X-API-Key: your-key" \+ -d '{+ "role": "user",+ "parts": [+ {"type": "text", "text": "Remember this studio layout."},+ {"type": "image_url", "image_url": {"url": "https://example.com/studio.png", "detail": "auto"}}+ ]+ }'```**Python SDK**```pythonimport openviking as ov-from openviking.message import TextPart, ContextPart+from openviking.message import TextPart, ImagePart, ContextPartclient = ov.Client(base_url="http://localhost:1933", api_key="your-key")@@ -691,6 +707,16 @@ await client.add_message()])++# Parts mode: Add user message with an image URL+await client.add_message(+ session_id="a1b2c3d4",+ role="user",+ parts=[+ TextPart(text="Remember this studio layout."),+ ImagePart(url="https://example.com/studio.png", detail="auto"),+ ]+)```**CLI**diff --git a/docs/en/concepts/08-session.md b/docs/en/concepts/08-session.mdindex d547af250..bfac42c20 100644--- a/docs/en/concepts/08-session.md+++ b/docs/en/concepts/08-session.md@@ -38,6 +38,14 @@ session.add_message(ContextPart(uri="viking://user/memories/profile.md"),])++session.add_message(+ "user",+ [+ TextPart("Remember this studio layout."),+ ImagePart(url="https://example.com/studio.png", detail="auto"),+ ]+)```### used@@ -90,6 +98,7 @@ class Message:| Type | Description ||------|-------------|| `TextPart` | Text content |+| `ImagePart` | Image URL content. During memory extraction, OpenViking can describe it with the configured VLM. || `ContextPart` | Context reference (URI + abstract) || `ToolPart` | Tool call (input + output) |diff --git a/docs/zh/concepts/08-session.md b/docs/zh/concepts/08-session.mdindex 982b07bb0..848edb4e9 100644--- a/docs/zh/concepts/08-session.md+++ b/docs/zh/concepts/08-session.md@@ -38,6 +38,14 @@ session.add_message(ContextPart(uri="viking://user/memories/profile.md"),])++session.add_message(+ "user",+ [+ TextPart("Remember this studio layout."),+ ImagePart(url="https://example.com/studio.png", detail="auto"),+ ]+)```### used@@ -90,6 +98,7 @@ class Message:| 类型 | 说明 ||------|------|| `TextPart` | 文本内容 |+| `ImagePart` | 图片 URL 内容。记忆提取时,OpenViking 可以使用已配置的 VLM 将其描述为文本。 || `ContextPart` | 上下文引用(URI + 摘要) || `ToolPart` | 工具调用(输入 + 输出) |diff --git a/openviking/async_client.py b/openviking/async_client.pyindex b02f4a6ff..c4365171b 100644--- a/openviking/async_client.py+++ b/openviking/async_client.py@@ -195,7 +195,7 @@ class AsyncOpenViking:session_id: Session IDrole: Message role ("user" or "assistant")content: Text content (simple mode)- parts: Parts array (full Part support: TextPart, ContextPart, ToolPart)+ parts: Parts array (full Part support: TextPart, ContextPart, ImagePart, ToolPart)created_at: Message creation time (ISO format string)peer_id: Optional stable interaction peer identity.diff --git a/openviking/client/session.py b/openviking/client/session.pyindex 786ddb9d7..b659aaff2 100644--- a/openviking/client/session.py+++ b/openviking/client/session.py@@ -48,7 +48,7 @@ class Session:Args:role: Message role (e.g., "user", "assistant")content: Text content (simple mode)- parts: Parts list (TextPart, ContextPart, ToolPart)+ parts: Parts list (TextPart, ContextPart, ImagePart, ToolPart)created_at: Message creation time (ISO format string). If not provided, current time is used.peer_id: Optional stable interaction peer identity.diff --git a/openviking/message/__init__.py b/openviking/message/__init__.pyindex 8dbe7a3c0..e0ff78c4c 100644--- a/openviking/message/__init__.py+++ b/openviking/message/__init__.py@@ -6,12 +6,19 @@ Message = role + parts"""from openviking.message.message import Message-from openviking.message.part import ContextPart, Part, TextPart, ToolPart+from openviking.message.part import (+ ContextPart,+ ImagePart,+ Part,+ TextPart,+ ToolPart,+)__all__ = ["Message","Part","TextPart","ContextPart",+ "ImagePart","ToolPart",]diff --git a/openviking/message/message.py b/openviking/message/message.pyindex 508096014..97cb73e09 100644--- a/openviking/message/message.py+++ b/openviking/message/message.py@@ -11,7 +11,7 @@ from datetime import datetime, timezonefrom typing import List, Literal, Optionalfrom openviking.core.peer_id import normalize_peer_id-from openviking.message.part import ContextPart, Part, TextPart, ToolPart+from openviking.message.part import ContextPart, ImagePart, Part, TextPart, ToolPartfrom openviking.utils.token_estimation import estimate_text_tokens@@ -40,6 +40,8 @@ class Message:Counts fields that actually appear in the assembled prompt:- TextPart.text: always emitted- ContextPart.abstract: injected as text (uri is not sent to the model)+ - ImagePart: not counted here; image captioning/model usage is tracked+ by the VLM call that converts images into text for extraction- ToolPart: tool_id (appears in toolUse.id / toolResult.toolCallId),tool_name, tool_input (JSON), tool_output@@ -93,6 +95,14 @@ class Message:"context_type": part.context_type,"abstract": part.abstract,}+ elif isinstance(part, ImagePart):+ image_url = {"url": part.url}+ if part.detail is not None:+ image_url["detail"] = part.detail+ return {+ "type": part.type,+ "image_url": image_url,+ }elif isinstance(part, ToolPart):d = {"type": part.type,@@ -168,6 +178,18 @@ class Message:abstract=p.get("abstract", ""),))+ elif p["type"] == "image_url":+ image_url = p.get("image_url")+ url = ""+ detail = None+ if isinstance(image_url, dict):+ url = str(image_url.get("url", "") or "")+ detail = image_url.get("detail")+ elif isinstance(image_url, str):+ url = image_url+ if not url.strip():+ raise ValueError("image_url part requires a non-empty URL")+ parts.append(ImagePart(url=url, detail=detail))elif p["type"] == "tool":parts.append(ToolPart(diff --git a/openviking/message/part.py b/openviking/message/part.pyindex 36d849368..d738e4836 100644--- a/openviking/message/part.py+++ b/openviking/message/part.py@@ -6,7 +6,7 @@ Message consists of multiple Parts, each Part has different type and purpose."""from dataclasses import dataclass-from typing import Literal, Optional, Union+from typing import Any, Dict, Literal, Optional, Union@dataclass@@ -30,6 +30,15 @@ class ContextPart:abstract: str = ""+@dataclass+class ImagePart:+ """Image URL component compatible with OpenAI-style message content."""++ type: Literal["image_url"] = "image_url"+ url: str = ""+ detail: Optional[str] = None++@dataclassclass ToolPart:"""Tool call component (references tool file within session).@@ -65,10 +74,19 @@ class ToolPart:tool_output_group_budget_chars: Optional[int] = None-Part = Union[TextPart, ContextPart, ToolPart]+Part = Union[TextPart, ContextPart, ImagePart, ToolPart]-def part_from_dict(data: dict) -> Part:+def _parse_image_url_payload(data: Dict[str, Any]) -> tuple[str, Optional[str]]:+ image_url = data.get("image_url")+ if isinstance(image_url, dict):+ return str(image_url.get("url", "") or ""), image_url.get("detail")+ if isinstance(image_url, str):+ return image_url, None+ return "", None+++def part_from_dict(data: Dict[str, Any]) -> Part:"""Convert a dict to a Part object.Args:@@ -86,6 +104,14 @@ def part_from_dict(data: dict) -> Part:context_type=data.get("context_type", "memory"),abstract=data.get("abstract", ""),)+ elif part_type == "image_url":+ url, detail = _parse_image_url_payload(data)+ if not url.strip():+ raise ValueError("image_url part requires a non-empty URL")+ return ImagePart(+ url=url,+ detail=detail,+ )elif part_type == "tool":return ToolPart(tool_id=data.get("tool_id", ""),diff --git a/openviking/server/routers/sessions.py b/openviking/server/routers/sessions.pyindex 3bdf554e4..bc173b98a 100644--- a/openviking/server/routers/sessions.py+++ b/openviking/server/routers/sessions.py@@ -4,7 +4,7 @@from typing import Any, Dict, List, Literal, Optional-from fastapi import APIRouter, Body, Depends, Path, Query, Request+from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query, Requestfrom pydantic import BaseModel, Field, model_validatorfrom openviking.core.path_variables import resolve_path_variables@@ -129,21 +129,37 @@ class CreateSessionRequest(BaseModel):def _resolve_message_parts(msg_request: AddMessageRequest) -> List[Part]:"""Resolve parts from an AddMessageRequest, handling path variables."""if msg_request.parts is not None:- resolved_parts = []- for p in msg_request.parts:- part_copy = dict(p)- if part_copy.get("type") == "context" and "uri" in part_copy:- part_copy["uri"] = resolve_path_variables(part_copy["uri"])- if part_copy.get("type") == "tool":- if "tool_uri" in part_copy:- part_copy["tool_uri"] = resolve_path_variables(part_copy["tool_uri"])- if "skill_uri" in part_copy:- part_copy["skill_uri"] = resolve_path_variables(part_copy["skill_uri"])- resolved_parts.append(part_copy)- return [part_from_dict(p) for p in resolved_parts]+ return [_part_request_to_part(p) for p in msg_request.parts]return [TextPart(text=msg_request.content or "")]+def _part_request_to_part(raw_part: Dict[str, Any]) -> Part:+ """Convert request part payload into an internal Part."""+ if not isinstance(raw_part, dict):+ return TextPart(text=str(raw_part))++ part_copy = dict(raw_part)+ if part_copy.get("type") == "context" and "uri" in part_copy:+ part_copy["uri"] = resolve_path_variables(part_copy["uri"])+ if part_copy.get("type") == "tool":+ if "tool_uri" in part_copy:+ part_copy["tool_uri"] = resolve_path_variables(part_copy["tool_uri"])+ if "skill_uri" in part_copy:+ part_copy["skill_uri"] = resolve_path_variables(part_copy["skill_uri"])+ if part_copy.get("type") == "image_url":+ image_url = part_copy.get("image_url")+ if isinstance(image_url, dict) and "url" in image_url:+ image_url = dict(image_url)+ image_url["url"] = resolve_path_variables(image_url["url"])+ part_copy["image_url"] = image_url+ elif isinstance(image_url, str):+ part_copy["image_url"] = resolve_path_variables(image_url)+ try:+ return part_from_dict(part_copy)+ except ValueError as exc:+ raise HTTPException(status_code=400, detail=str(exc)) from exc++def _to_jsonable(value: Any) -> Any:"""Convert internal objects (e.g. Context) into JSON-serializable values."""to_dict = getattr(value, "to_dict", None)diff --git a/openviking/session/compressor_v2.py b/openviking/session/compressor_v2.pyindex fb55d20fb..9ba691b22 100644--- a/openviking/session/compressor_v2.py+++ b/openviking/session/compressor_v2.py@@ -309,6 +309,7 @@ class SessionCompressorV2:viking_fs=viking_fs,transaction_handle=transaction_handle,)+ await context_provider.prepare_extraction_messages()extract_context = context_provider.get_extract_context()isolation_handler = MemoryIsolationHandler(ctx,@@ -717,6 +718,8 @@ class SessionCompressorV2:# Use the provider's extraction context so prompt ranges and memory# rendering resolve against the same message list.+ await provider.prepare_extraction_messages()+extract_context = provider.get_extract_context()isolation_handler = MemoryIsolationHandler(ctx,diff --git a/openviking/session/memory/core.py b/openviking/session/memory/core.pyindex ba22734bd..77b89599b 100644--- a/openviking/session/memory/core.py+++ b/openviking/session/memory/core.py@@ -27,6 +27,12 @@ class ExtractContextProvider(ABC):"""pass+ async def prepare_extraction_messages(self) -> None:+ """+ 在构建 prompt、ranges 和 ExtractContext 之前准备 extraction-only messages。+ """+ return None+@abstractmethodasync def prefetch(self,diff --git a/openviking/session/memory/session_extract_context_provider.py b/openviking/session/memory/session_extract_context_provider.pyindex 3afed47fe..fb464af82 100644--- a/openviking/session/memory/session_extract_context_provider.py+++ b/openviking/session/memory/session_extract_context_provider.py@@ -10,7 +10,7 @@ import jsonimport osfrom typing import TYPE_CHECKING, Any, Dict, List, Optional-from openviking.message.part import ToolPart+from openviking.message.part import TextPart, ToolPartfrom openviking.prompts.manager import PromptManagerfrom openviking.server.identity import RequestContext, ToolContextfrom openviking.session.memory.core import ExtractContextProvider@@ -29,6 +29,9 @@ from openviking.session.memory.tools import (get_tool,)from openviking.session.memory.utils.uri import render_template+from openviking.session.memory.vision_message_normalizer import (+ replace_image_parts_with_descriptions,+)from openviking.storage.viking_fs import VikingFSfrom openviking.telemetry import tracerfrom openviking.utils.time_utils import parse_iso_datetime@@ -58,7 +61,7 @@ class SessionExtractContextProvider(ExtractContextProvider):viking_fs: VikingFS = None,transaction_handle=None,):- self.messages = messages+ self.messages = list(messages) if isinstance(messages, list) else messagesself.latest_archive_overview = latest_archive_overviewself._output_language = self._detect_language()self._registry = None # 延迟加载@@ -74,6 +77,8 @@ class SessionExtractContextProvider(ExtractContextProvider):self._viking_fs = viking_fsself._transaction_handle = transaction_handleself._link_enabled = config.memory.link_enabled if config.memory else False+ self._vision_messages_prepared = False+ self._vision_vlm = None@propertydef read_file_contents(self) -> Dict[str, MemoryFile]:@@ -104,9 +109,31 @@ class SessionExtractContextProvider(ExtractContextProvider):)return self._extract_context+ async def prepare_extraction_messages(self) -> None:+ """Prepare extraction-only messages before ranges and prompts are built."""+ if self._vision_messages_prepared:+ return+ if isinstance(self.messages, list):+ self.messages = await replace_image_parts_with_descriptions(+ self.messages,+ get_vlm=self._get_vision_vlm,+ logger=logger,+ )+ self._extract_context = None+ self._output_language = self._detect_language()+ self._vision_messages_prepared = True++ def _get_vision_vlm(self):+ if self._vision_vlm is not None:+ return self._vision_vlm+ vlm_config = get_openviking_config().vlm+ if not (vlm_config and vlm_config.is_available()):+ return None+ self._vision_vlm = vlm_config.get_vlm_instance()+ return self._vision_vlm+def _detect_language(self) -> str:"""检测输出语言"""- from openviking.message.part import TextPartfrom openviking.session.memory.utils import resolve_output_languageuser_text_parts = []diff --git a/openviking/session/memory/vision_message_normalizer.py b/openviking/session/memory/vision_message_normalizer.pynew file mode 100644index 000000000..3cb73c1a0--- /dev/null+++ b/openviking/session/memory/vision_message_normalizer.py@@ -0,0 +1,108 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Normalize image messages into extraction-friendly text messages."""++from collections.abc import Callable+from typing import Any, Dict, List++from openviking.message import Message+from openviking.message.part import ImagePart, TextPart++IMAGE_DESCRIPTION_PROMPT = (+ "Describe this image for later memory extraction. Focus on durable, user-relevant "+ "details such as visible people, objects, places, actions, text, dates, and other "+ "facts that may matter in future conversations. Return only the description."+)+++def message_has_image_part(message: Message) -> bool:+ return any(isinstance(part, ImagePart) for part in getattr(message, "parts", []))+++def image_part_to_openai_content(part: ImagePart) -> Dict[str, Any]:+ image_url: Dict[str, Any] = {"url": part.url}+ if part.detail is not None:+ image_url["detail"] = part.detail+ return {"type": "image_url", "image_url": image_url}+++def build_vision_description_messages(message: Message) -> List[Dict[str, Any]]:+ content: List[Dict[str, Any]] = [{"type": "text", "text": IMAGE_DESCRIPTION_PROMPT}]+ for part in getattr(message, "parts", []):+ if isinstance(part, TextPart) and part.text:+ content.append({"type": "text", "text": part.text})+ elif isinstance(part, ImagePart):+ content.append(image_part_to_openai_content(part))+ return [{"role": "user", "content": content}]+++def fallback_image_description(message: Message) -> str:+ return ""+++def normalize_vision_response(response: Any) -> str:+ content = getattr(response, "content", response)+ return str(content or "").strip()+++def _original_text_parts(message: Message) -> List[TextPart]:+ return [+ TextPart(text=part.text)+ for part in getattr(message, "parts", [])+ if isinstance(part, TextPart) and part.text+ ]+++async def describe_image_message(+ message: Message,+ *,+ vlm: Any,+ logger: Any = None,+) -> str:+ if vlm is None:+ return fallback_image_description(message)++ try:+ response = await vlm.get_vision_completion_async(+ messages=build_vision_description_messages(message),+ thinking=False,+ )+ description = normalize_vision_response(response)+ return description or fallback_image_description(message)+ except Exception as exc:+ if logger is not None:+ logger.warning("Failed to describe image message %s: %s", message.id, exc)+ return fallback_image_description(message)+++async def replace_image_parts_with_descriptions(+ messages: List[Message],+ *,+ get_vlm: Callable[[], Any],+ logger: Any = None,+) -> List[Message]:+ prepared_messages: List[Message] = []+ for message in messages:+ if not message_has_image_part(message):+ prepared_messages.append(message)+ continue++ description = await describe_image_message(+ message,+ vlm=get_vlm(),+ logger=logger,+ )+ parts = _original_text_parts(message)+ if description:+ parts.append(TextPart(text=f"[Image description]: {description}"))+ if parts:+ prepared_messages.append(+ Message(+ id=message.id,+ role=message.role,+ parts=parts,+ peer_id=message.peer_id,+ created_at=message.created_at,+ )+ )+ return prepared_messagesdiff --git a/openviking/sync_client.py b/openviking/sync_client.pyindex 55ee66d4d..80017c4d7 100644--- a/openviking/sync_client.py+++ b/openviking/sync_client.py@@ -107,7 +107,7 @@ class SyncOpenViking:session_id: Session IDrole: Message role ("user" or "assistant")content: Text content (simple mode)- parts: Parts array (full Part support: TextPart, ContextPart, ToolPart)+ parts: Parts array (full Part support: TextPart, ContextPart, ImagePart, ToolPart)created_at: Message creation time (ISO format string). If not provided, current time is used.peer_id: Optional stable interaction peer identity.diff --git a/openviking_cli/client/base.py b/openviking_cli/client/base.pyindex f07b15b3e..47c8a2256 100644--- a/openviking_cli/client/base.py+++ b/openviking_cli/client/base.py@@ -304,7 +304,7 @@ class BaseClient(ABC):session_id: Session IDrole: Message role ("user" or "assistant")content: Text content (simple mode)- parts: Parts array (full Part support: TextPart, ContextPart, ToolPart)+ parts: Parts array (full Part support: TextPart, ContextPart, ImagePart, ToolPart)created_at: Message creation time (ISO format string)peer_id: Optional stable interaction peer identity.telemetry: Whether to attach operation telemetry data to the result.diff --git a/openviking_cli/client/sync_http.py b/openviking_cli/client/sync_http.pyindex f66a59b1d..474287bcb 100644--- a/openviking_cli/client/sync_http.py+++ b/openviking_cli/client/sync_http.py@@ -143,7 +143,7 @@ class SyncHTTPClient:session_id: Session IDrole: Message role ("user" or "assistant")content: Text content (simple mode)- parts: Parts array (full Part support: TextPart, ContextPart, ToolPart)+ parts: Parts array (full Part support: TextPart, ContextPart, ImagePart, ToolPart)created_at: Message creation time (ISO format string)peer_id: Optional stable interaction peer identity.diff --git a/tests/client/test_rebuild_clients.py b/tests/client/test_rebuild_clients.pyindex df78bdc1d..3d7f153ba 100644--- a/tests/client/test_rebuild_clients.py+++ b/tests/client/test_rebuild_clients.py@@ -11,6 +11,7 @@ import openviking_cli.client.http as http_moduleimport openviking_cli.utils.async_utils as async_utilsfrom openviking import AsyncOpenViking, SyncOpenVikingfrom openviking.client.local import LocalClient+from openviking.message import ImagePart, TextPartfrom openviking_cli.client.http import AsyncHTTPClientfrom openviking_cli.client.sync_http import SyncHTTPClientfrom openviking_cli.utils.config import OPENVIKING_CLI_CONFIG_ENV@@ -180,6 +181,50 @@ async def test_local_client_batch_add_messages_forwards_to_session():assert fake_session.messages[1]["parts"][0].text == "hi"+async def test_local_client_add_message_accepts_image_parts():+ class FakeSession:+ def __init__(self):+ self.messages = []++ def add_message(self, role, parts, peer_id=None, created_at=None):+ self.messages.append(+ {+ "role": role,+ "parts": parts,+ "peer_id": peer_id,+ "created_at": created_at,+ }+ )++ fake_session = FakeSession()++ class FakeSessions:+ async def get(self, session_id, ctx, auto_create=False):+ assert session_id == "image-session"+ assert ctx is client._ctx+ assert auto_create is True+ return fake_session++ client = LocalClient.__new__(LocalClient)+ client._service = SimpleNamespace(sessions=FakeSessions())+ client._ctx = SimpleNamespace(user=SimpleNamespace(user_id="user-1"))++ result = await LocalClient.add_message(+ client,+ "image-session",+ "user",+ parts=[+ {"type": "text", "text": "Look at this"},+ {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}},+ ],+ )++ assert result == {"session_id": "image-session", "message_count": 1}+ assert isinstance(fake_session.messages[0]["parts"][0], TextPart)+ assert isinstance(fake_session.messages[0]["parts"][1], ImagePart)+ assert fake_session.messages[0]["parts"][1].url == "https://example.com/image.png"++async def test_async_http_client_batch_add_messages_posts_batch_payload():client = AsyncHTTPClient(url="http://localhost:1933")fake_http = SimpleNamespace(post=AsyncMock(return_value=object()))diff --git a/tests/server/test_api_sessions.py b/tests/server/test_api_sessions.pyindex da9291491..628ad74cd 100644--- a/tests/server/test_api_sessions.py+++ b/tests/server/test_api_sessions.py@@ -12,7 +12,7 @@ import pytestfrom fastapi import FastAPIfrom starlette.requests import Request-from openviking.message import Message, TextPart+from openviking.message import ImagePart, Message, TextPartfrom openviking.server.app import create_appfrom openviking.server.config import ServerConfig, ToolOutputExternalizationConfigfrom openviking.server.dependencies import set_service@@ -367,6 +367,169 @@ async def test_add_message(client: httpx.AsyncClient):assert body["result"]["message_count"] == 1+async def test_add_message_accepts_image_part(client: httpx.AsyncClient, service):+ create_resp = await client.post("/api/v1/sessions", json={})+ session_id = create_resp.json()["result"]["session_id"]++ resp = await client.post(+ f"/api/v1/sessions/{session_id}/messages",+ json=_message_request(+ "user",+ parts=[+ {+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/image.png",+ "detail": "auto",+ },+ }+ ],+ ),+ )++ assert resp.status_code == 200+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)+ session = service.sessions.session(ctx, session_id)+ await session.load()+ assert isinstance(session.messages[0].parts[0], ImagePart)+ assert session.messages[0].parts[0].url == "https://example.com/image.png"+ assert session.messages[0].parts[0].detail == "auto"++ context_resp = await client.get(f"/api/v1/sessions/{session_id}/context")+ assert context_resp.status_code == 200+ assert context_resp.json()["result"]["messages"][0]["parts"][0] == {+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/image.png",+ "detail": "auto",+ },+ }+++async def test_add_message_resolves_image_part_url_path_variables(+ client: httpx.AsyncClient,+ service,+ monkeypatch,+):+ monkeypatch.setattr(+ "openviking.server.routers.sessions.resolve_path_variables",+ lambda value: value.replace("{calendar:today}", "2026/06/15"),+ )+ create_resp = await client.post("/api/v1/sessions", json={})+ session_id = create_resp.json()["result"]["session_id"]++ resp = await client.post(+ f"/api/v1/sessions/{session_id}/messages",+ json=_message_request(+ "user",+ parts=[+ {+ "type": "image_url",+ "image_url": {"url": "viking://resources/images/{calendar:today}/photo.png"},+ }+ ],+ ),+ )++ assert resp.status_code == 200+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)+ session = service.sessions.session(ctx, session_id)+ await session.load()+ assert isinstance(session.messages[0].parts[0], ImagePart)+ assert session.messages[0].parts[0].url == "viking://resources/images/2026/06/15/photo.png"+++async def test_add_message_accepts_mixed_parts(client: httpx.AsyncClient, service):+ create_resp = await client.post("/api/v1/sessions", json={})+ session_id = create_resp.json()["result"]["session_id"]++ resp = await client.post(+ f"/api/v1/sessions/{session_id}/messages",+ json=_message_request(+ "user",+ parts=[+ {"type": "text", "text": "Look at this"},+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png"},+ },+ ],+ ),+ )++ assert resp.status_code == 200+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)+ session = service.sessions.session(ctx, session_id)+ await session.load()+ assert isinstance(session.messages[0].parts[0], TextPart)+ assert session.messages[0].parts[0].text == "Look at this"+ assert isinstance(session.messages[0].parts[1], ImagePart)+ assert session.messages[0].parts[1].url == "https://example.com/image.png"+++async def test_add_message_rejects_image_part_without_url(client: httpx.AsyncClient):+ create_resp = await client.post("/api/v1/sessions", json={})+ session_id = create_resp.json()["result"]["session_id"]++ resp = await client.post(+ f"/api/v1/sessions/{session_id}/messages",+ json=_message_request(+ "user",+ parts=[{"type": "image_url", "image_url": {}}],+ ),+ )++ assert resp.status_code == 400+ assert "image_url part requires a non-empty URL" in resp.text+++async def test_add_message_rejects_openai_style_image_content(client: httpx.AsyncClient):+ create_resp = await client.post("/api/v1/sessions", json={})+ session_id = create_resp.json()["result"]["session_id"]++ resp = await client.post(+ f"/api/v1/sessions/{session_id}/messages",+ json={+ "role": "user",+ "content": {+ "type": "image_url",+ },+ },+ )++ assert resp.status_code == 400+++async def test_batch_add_message_accepts_mixed_parts(client: httpx.AsyncClient, service):+ create_resp = await client.post("/api/v1/sessions", json={})+ session_id = create_resp.json()["result"]["session_id"]++ resp = await client.post(+ f"/api/v1/sessions/{session_id}/messages/batch",+ json={+ "messages": [+ _message_request(+ "user",+ parts=[+ {"type": "text", "text": "Look at this"},+ {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png"},+ },+ ],+ )+ ]+ },+ )++ assert resp.status_code == 200+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)+ session = service.sessions.session(ctx, session_id)+ await session.load()+ assert isinstance(session.messages[0].parts[0], TextPart)+ assert isinstance(session.messages[0].parts[1], ImagePart)++async def test_add_message_splits_tool_result_aggregate(client: httpx.AsyncClient):create_resp = await client.post("/api/v1/sessions", json={})session_id = create_resp.json()["result"]["session_id"]diff --git a/tests/session/memory/test_compressor_v2.py b/tests/session/memory/test_compressor_v2.pyindex 8dec9e1f6..79143a807 100644--- a/tests/session/memory/test_compressor_v2.py+++ b/tests/session/memory/test_compressor_v2.py@@ -622,6 +622,9 @@ class TestCompressorV2:return uriclass DummyProvider:+ async def prepare_extraction_messages(self):+ pass+def get_memory_schemas(self, _ctx):return []@@ -828,6 +831,9 @@ class TestExtractLoopPatchRepair:def __init__(self):self.extract_context = ExtractContext([])+ async def prepare_extraction_messages(self):+ pass+def get_memory_schemas(self, _ctx):return [schema]@@ -925,6 +931,9 @@ class TestExtractLoopPatchRepair:def __init__(self):self.extract_context = ExtractContext([])+ async def prepare_extraction_messages(self):+ pass+def get_memory_schemas(self, _ctx):return [schema]@@ -1021,6 +1030,9 @@ class TestExtractLoopPatchRepair:def __init__(self):self.extract_context = ExtractContext([])+ async def prepare_extraction_messages(self):+ pass+def get_memory_schemas(self, _ctx):return [schema]diff --git a/tests/session/memory/test_memory_react_system_prompt.py b/tests/session/memory/test_memory_react_system_prompt.pyindex 61bf63eed..e14f5e274 100644--- a/tests/session/memory/test_memory_react_system_prompt.py+++ b/tests/session/memory/test_memory_react_system_prompt.py@@ -4,8 +4,9 @@Test that provider instruction correctly instructs LLM."""-from openviking.message import Message, TextPart, ToolPart+from openviking.message import ImagePart, Message, TextPart, ToolPartfrom openviking.session.memory.session_extract_context_provider import SessionExtractContextProvider+from openviking.session.memory.vision_message_normalizer import IMAGE_DESCRIPTION_PROMPTclass TestProviderInstruction:@@ -165,3 +166,110 @@ class TestSkillToolCallExposure:provider = SessionExtractContextProvider(messages=messages)assert provider._detect_language() == "zh-CN"++ async def test_prepare_extraction_messages_replaces_image_part_with_vlm_description(self):+ class FakeVisionVLM:+ def __init__(self):+ self.messages = None++ async def get_vision_completion_async(self, **kwargs):+ self.messages = kwargs.get("messages")+ return "A high level image description."++ messages = [+ Message(+ id="m1",+ role="user",+ parts=[+ TextPart("Please remember this image."),+ ImagePart(url="https://example.com/image.png", detail="auto"),+ ],+ )+ ]+ provider = SessionExtractContextProvider(messages=messages)+ fake_vlm = FakeVisionVLM()+ provider._vision_vlm = fake_vlm++ await provider.prepare_extraction_messages()+ prompt_message = provider._build_conversation_message()++ assert "Please remember this image." in prompt_message["content"]+ assert "A high level image description." in prompt_message["content"]+ assert "https://example.com/image.png" not in prompt_message["content"]+ assert fake_vlm.messages == [+ {+ "role": "user",+ "content": [+ {+ "type": "text",+ "text": IMAGE_DESCRIPTION_PROMPT,+ },+ {"type": "text", "text": "Please remember this image."},+ {+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/image.png",+ "detail": "auto",+ },+ },+ ],+ }+ ]++ async def test_prepare_extraction_messages_skips_image_only_without_description(self):+ messages = [+ Message(+ id="m1",+ role="user",+ parts=[ImagePart(url="https://example.com/private-family-photo.png")],+ )+ ]+ provider = SessionExtractContextProvider(messages=messages)+ provider._vision_vlm = None++ await provider.prepare_extraction_messages()+ prompt_message = provider._build_conversation_message()++ assert "[Image description]" not in prompt_message["content"]+ assert "https://example.com/private-family-photo.png" not in prompt_message["content"]++ async def test_prepare_extraction_messages_keeps_text_when_image_is_undescribed(self):+ messages = [+ Message(+ id="m1",+ role="user",+ parts=[+ TextPart("Please remember the surrounding text."),+ ImagePart(url="https://example.com/private-family-photo.png"),+ ],+ )+ ]+ provider = SessionExtractContextProvider(messages=messages)+ provider._vision_vlm = None++ await provider.prepare_extraction_messages()+ prompt_message = provider._build_conversation_message()++ assert "Please remember the surrounding text." in prompt_message["content"]+ assert "[Image description]" not in prompt_message["content"]+ assert "https://example.com/private-family-photo.png" not in prompt_message["content"]++ async def test_prepare_extraction_messages_does_not_replace_caller_message_list(self):+ messages = [+ Message(+ id="m1",+ role="user",+ parts=[+ TextPart("Please remember this image."),+ ImagePart(url="https://example.com/image.png"),+ ],+ )+ ]+ provider = SessionExtractContextProvider(messages=messages)+ provider._vision_vlm = None++ await provider.prepare_extraction_messages()++ assert len(messages) == 1+ assert any(isinstance(part, ImagePart) for part in messages[0].parts)+ assert provider.messages is not messagesdiff --git a/tests/unit/test_message.py b/tests/unit/test_message.pyindex 30d803c1f..987bd4f2a 100644--- a/tests/unit/test_message.py+++ b/tests/unit/test_message.py@@ -6,7 +6,9 @@import jsonfrom datetime import datetime, timezone-from openviking.message import ContextPart, Message, TextPart, ToolPart+import pytest++from openviking.message import ContextPart, ImagePart, Message, TextPart, ToolPartfrom openviking.message.part import part_from_dict@@ -108,6 +110,29 @@ class TestContextPart:assert part.context_type == "resource"+class TestImagePart:+ """Test ImagePart dataclass."""++ def test_default_values(self):+ """Test default values."""+ part = ImagePart()++ assert part.url == ""+ assert part.detail is None+ assert part.type == "image_url"++ def test_custom_values(self):+ """Test custom values."""+ part = ImagePart(+ url="https://example.com/image.png",+ detail="auto",+ )++ assert part.url == "https://example.com/image.png"+ assert part.detail == "auto"+ assert part.type == "image_url"++class TestToolPart:"""Test ToolPart dataclass."""@@ -258,6 +283,37 @@ class TestPartFromDict:assert part.tool_output_preview_chars == 100assert part.tool_output_sha256 == "abc123"+ def test_image_part_from_openai_style_dict(self):+ """Test creating ImagePart from OpenAI-style image_url dict."""+ data = {+ "type": "image_url",+ "image_url": {"url": "https://example.com/image.png", "detail": "high"},+ }++ part = part_from_dict(data)++ assert isinstance(part, ImagePart)+ assert part.url == "https://example.com/image.png"+ assert part.detail == "high"++ def test_image_part_rejects_flat_dict(self):+ """OpenAI-style image_url payloads require the nested image_url shape."""+ data = {"type": "image_url", "url": "https://example.com/image.png"}++ with pytest.raises(ValueError, match="image_url part requires a non-empty URL"):+ part_from_dict(data)++ def test_image_part_rejects_missing_url(self):+ """Test image_url parts require a non-empty URL."""+ data = {"type": "image_url", "image_url": {}}++ try:+ part_from_dict(data)+ except ValueError as exc:+ assert "image_url part requires a non-empty URL" in str(exc)+ else:+ raise AssertionError("Expected ValueError for missing image URL")+def test_unknown_type_defaults_to_text(self):"""Test unknown type defaults to TextPart."""data = {"type": "unknown", "value": "something"}@@ -473,6 +529,30 @@ class TestMessageToDict:assert d["parts"][0]["tool_output_preview_chars"] == 100assert d["parts"][0]["tool_output_externalized_reason"] == "single_threshold"+ def test_to_dict_with_image_part(self):+ """Test to_dict with ImagePart."""+ msg = Message(+ id="msg-1",+ role="user",+ parts=[+ TextPart(text="Look at this"),+ ImagePart(+ url="https://example.com/image.png",+ detail="auto",+ ),+ ],+ )++ d = msg.to_dict()++ assert d["parts"][1] == {+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/image.png",+ "detail": "auto",+ },+ }+def test_to_dict_timestamp_format(self):"""Test timestamp format in to_dict."""now = datetime(2026, 3, 26, 10, 30, 0, tzinfo=timezone.utc)@@ -561,6 +641,31 @@ class TestMessageFromDict:assert msg.parts[0].tool_output_preview_chars == 100assert msg.parts[0].tool_output_externalized_reason == "single_threshold"+ def test_from_dict_with_image_part(self):+ """Test from_dict with ImagePart."""+ d = {+ "id": "msg-1",+ "role": "user",+ "parts": [+ {"type": "text", "text": "Look at this"},+ {+ "type": "image_url",+ "image_url": {+ "url": "https://example.com/image.png",+ "detail": "auto",+ },+ },+ ],+ "created_at": "2026-03-26T10:30:00Z",+ }++ msg = Message.from_dict(d)++ assert isinstance(msg.parts[0], TextPart)+ assert isinstance(msg.parts[1], ImagePart)+ assert msg.parts[1].url == "https://example.com/image.png"+ assert msg.parts[1].detail == "auto"+def test_from_dict_supports_legacy_content_only_messages(self):"""Legacy messages with only content should load as a TextPart."""d = {