Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(session): Support image message extraction (#2578)

agenta4aefac1f793
diff --git a/benchmark/locomo/openviking/README.md b/benchmark/locomo/openviking/README.md
index 0e15a6e07..3352636f2 100644
--- a/benchmark/locomo/openviking/README.md
+++ b/benchmark/locomo/openviking/README.md
@@ -24,6 +24,13 @@ LoCoMo samples are imported under `viking://user/sample_{idx}/memories`.
Evaluation uses the same `sample_{idx}` user id, so import and eval must use the
same dataset order.
+LoCoMo image evidence is imported as text from `blip_caption` by default. This
+keeps benchmark runs compatible with older OpenViking servers and avoids making
+results depend on whether the configured VLM can download external image URLs.
+Use `--use-image-url` when you explicitly want to import structured
+`image_url` parts and let OpenViking's VLM describe images during memory
+extraction.
+
## Import
```bash
@@ -35,6 +42,8 @@ python benchmark/locomo/openviking/import_to_ov.py \
```
Use `--force-ingest` only when intentionally re-importing existing samples.
+Use `--use-image-url` only for multimodal import experiments; the default uses
+LoCoMo's `blip_caption` text.
## Smoke Test
diff --git a/benchmark/locomo/openviking/import_to_ov.py b/benchmark/locomo/openviking/import_to_ov.py
index 20752ab2e..178776d3c 100644
--- a/benchmark/locomo/openviking/import_to_ov.py
+++ b/benchmark/locomo/openviking/import_to_ov.py
@@ -171,19 +171,30 @@ def _normalize_image_urls(value: Any) -> List[str]:
return [str(value)]
-def build_openviking_message_parts(msg: Dict[str, Any]) -> List[Dict[str, Any]]:
+def build_openviking_message_parts(
+ msg: Dict[str, Any],
+ *,
+ use_image_url: bool = False,
+) -> List[Dict[str, Any]]:
"""Build OpenViking message parts from a LoCoMo message."""
parts: List[Dict[str, Any]] = [{"type": "text", "text": msg["text"]}]
caption = str(msg.get("blip_caption", "") or "")
for image_url in _normalize_image_urls(msg.get("img_url")):
- image_part: Dict[str, Any] = {
- "type": "image_url",
- "image_url": {"url": image_url},
- }
- if caption:
- image_part["caption"] = caption
- parts.append(image_part)
+ if use_image_url:
+ parts.append(
+ {
+ "type": "image_url",
+ "image_url": {"url": image_url},
+ }
+ )
+ else:
+ parts.append(
+ {
+ "type": "text",
+ "text": f"[Image URL: {image_url}; BLIP caption: {caption}]",
+ }
+ )
return parts
@@ -342,6 +353,7 @@ async def viking_ingest(
session_time: Optional[str] = None,
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
+ use_image_url: bool = False,
) -> Dict[str, int]:
"""Save messages to OpenViking via OpenViking SDK client.
Returns token usage dict with embedding and vlm token counts.
@@ -384,7 +396,10 @@ async def viking_ingest(
await client.add_message(
session_id=session_id,
role=msg["role"],
- parts=build_openviking_message_parts(msg),
+ parts=build_openviking_message_parts(
+ msg,
+ use_image_url=use_image_url,
+ ),
created_at=msg_created_at,
)
@@ -455,6 +470,7 @@ async def process_single_session(
meta.get("date_time"),
user_id=str(sample_id),
agent_id=str(sample_id),
+ use_image_url=args.use_image_url,
)
chunk_usage = result["token_usage"]
for key in token_usage:
@@ -787,6 +803,12 @@ def main():
default=False,
help="Clear all existing ingest records before running",
)
+ parser.add_argument(
+ "--use-image-url",
+ action="store_true",
+ default=False,
+ help="Import LoCoMo images as structured image_url parts. Default: false; use blip_caption text for compatibility.",
+ )
args = parser.parse_args()
# 确保输出目录存在
diff --git a/docs/en/api/05-sessions.md b/docs/en/api/05-sessions.md
index 0a61588ac..e2057cea7 100644
--- a/docs/en/api/05-sessions.md
+++ b/docs/en/api/05-sessions.md
@@ -556,10 +556,11 @@ ov session delete a1b2c3d4
#### 1. API Implementation Introduction
-Add a message to the session. Supports two modes: simple text mode and Parts mode (supporting text, context references, tool calls, etc.).
+Add a message to the session. Supports two modes: simple text mode and Parts mode (supporting text, image URLs, context references, tool calls, etc.).
**Part Types:**
- `TextPart`: Pure text content
+- `ImagePart`: OpenAI-style image URL content. During memory extraction, OpenViking can use the configured VLM to turn images into text descriptions.
- `ContextPart`: Context reference pointing to resources or memories
- `ToolPart`: Tool call and result
@@ -591,11 +592,14 @@ Add a message to the session. Supports two modes: simple text mode and Parts mod
**Part Types (Python SDK)**
```python
-from openviking.message import TextPart, ContextPart, ToolPart
+from openviking.message import TextPart, ImagePart, ContextPart, ToolPart
# Text content
TextPart(text="Hello, how can I help?")
+# Image URL content
+ImagePart(url="https://example.com/photo.png", detail="auto")
+
# Context reference
ContextPart(
uri="viking://resources/docs/auth/",
@@ -661,13 +665,25 @@ curl -X POST http://localhost:1933/api/v1/sessions/a1b2c3d4/messages \
{"type": "tool", "tool_id": "call_123", "tool_name": "search_web", "tool_input": {"query": "OAuth"}, "tool_status": "completed", "tool_output": "Results..."}
]
}'
+
+# Add user message with an image URL
+curl -X POST http://localhost:1933/api/v1/sessions/a1b2c3d4/messages \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{
+ "role": "user",
+ "parts": [
+ {"type": "text", "text": "Remember this studio layout."},
+ {"type": "image_url", "image_url": {"url": "https://example.com/studio.png", "detail": "auto"}}
+ ]
+ }'
```
**Python SDK**
```python
import openviking as ov
-from openviking.message import TextPart, ContextPart
+from openviking.message import TextPart, ImagePart, ContextPart
client = ov.Client(base_url="http://localhost:1933", api_key="your-key")
@@ -691,6 +707,16 @@ await client.add_message(
)
]
)
+
+# Parts mode: Add user message with an image URL
+await client.add_message(
+ session_id="a1b2c3d4",
+ role="user",
+ parts=[
+ TextPart(text="Remember this studio layout."),
+ ImagePart(url="https://example.com/studio.png", detail="auto"),
+ ]
+)
```
**CLI**
diff --git a/docs/en/concepts/08-session.md b/docs/en/concepts/08-session.md
index d547af250..bfac42c20 100644
--- a/docs/en/concepts/08-session.md
+++ b/docs/en/concepts/08-session.md
@@ -38,6 +38,14 @@ session.add_message(
ContextPart(uri="viking://user/memories/profile.md"),
]
)
+
+session.add_message(
+ "user",
+ [
+ TextPart("Remember this studio layout."),
+ ImagePart(url="https://example.com/studio.png", detail="auto"),
+ ]
+)
```
### used
@@ -90,6 +98,7 @@ class Message:
| Type | Description |
|------|-------------|
| `TextPart` | Text content |
+| `ImagePart` | Image URL content. During memory extraction, OpenViking can describe it with the configured VLM. |
| `ContextPart` | Context reference (URI + abstract) |
| `ToolPart` | Tool call (input + output) |
diff --git a/docs/zh/concepts/08-session.md b/docs/zh/concepts/08-session.md
index 982b07bb0..848edb4e9 100644
--- a/docs/zh/concepts/08-session.md
+++ b/docs/zh/concepts/08-session.md
@@ -38,6 +38,14 @@ session.add_message(
ContextPart(uri="viking://user/memories/profile.md"),
]
)
+
+session.add_message(
+ "user",
+ [
+ TextPart("Remember this studio layout."),
+ ImagePart(url="https://example.com/studio.png", detail="auto"),
+ ]
+)
```
### used
@@ -90,6 +98,7 @@ class Message:
| 类型 | 说明 |
|------|------|
| `TextPart` | 文本内容 |
+| `ImagePart` | 图片 URL 内容。记忆提取时,OpenViking 可以使用已配置的 VLM 将其描述为文本。 |
| `ContextPart` | 上下文引用(URI + 摘要) |
| `ToolPart` | 工具调用(输入 + 输出) |
diff --git a/openviking/async_client.py b/openviking/async_client.py
index b02f4a6ff..c4365171b 100644
--- a/openviking/async_client.py
+++ b/openviking/async_client.py
@@ -195,7 +195,7 @@ class AsyncOpenViking:
session_id: Session ID
role: Message role ("user" or "assistant")
content: Text content (simple mode)
- parts: Parts array (full Part support: TextPart, ContextPart, ToolPart)
+ parts: Parts array (full Part support: TextPart, ContextPart, ImagePart, ToolPart)
created_at: Message creation time (ISO format string)
peer_id: Optional stable interaction peer identity.
diff --git a/openviking/client/session.py b/openviking/client/session.py
index 786ddb9d7..b659aaff2 100644
--- a/openviking/client/session.py
+++ b/openviking/client/session.py
@@ -48,7 +48,7 @@ class Session:
Args:
role: Message role (e.g., "user", "assistant")
content: Text content (simple mode)
- parts: Parts list (TextPart, ContextPart, ToolPart)
+ parts: Parts list (TextPart, ContextPart, ImagePart, ToolPart)
created_at: Message creation time (ISO format string). If not provided, current time is used.
peer_id: Optional stable interaction peer identity.
diff --git a/openviking/message/__init__.py b/openviking/message/__init__.py
index 8dbe7a3c0..e0ff78c4c 100644
--- a/openviking/message/__init__.py
+++ b/openviking/message/__init__.py
@@ -6,12 +6,19 @@ Message = role + parts
"""
from openviking.message.message import Message
-from openviking.message.part import ContextPart, Part, TextPart, ToolPart
+from openviking.message.part import (
+ ContextPart,
+ ImagePart,
+ Part,
+ TextPart,
+ ToolPart,
+)
__all__ = [
"Message",
"Part",
"TextPart",
"ContextPart",
+ "ImagePart",
"ToolPart",
]
diff --git a/openviking/message/message.py b/openviking/message/message.py
index 508096014..97cb73e09 100644
--- a/openviking/message/message.py
+++ b/openviking/message/message.py
@@ -11,7 +11,7 @@ from datetime import datetime, timezone
from typing import List, Literal, Optional
from openviking.core.peer_id import normalize_peer_id
-from openviking.message.part import ContextPart, Part, TextPart, ToolPart
+from openviking.message.part import ContextPart, ImagePart, Part, TextPart, ToolPart
from openviking.utils.token_estimation import estimate_text_tokens
@@ -40,6 +40,8 @@ class Message:
Counts fields that actually appear in the assembled prompt:
- TextPart.text: always emitted
- ContextPart.abstract: injected as text (uri is not sent to the model)
+ - ImagePart: not counted here; image captioning/model usage is tracked
+ by the VLM call that converts images into text for extraction
- ToolPart: tool_id (appears in toolUse.id / toolResult.toolCallId),
tool_name, tool_input (JSON), tool_output
@@ -93,6 +95,14 @@ class Message:
"context_type": part.context_type,
"abstract": part.abstract,
}
+ elif isinstance(part, ImagePart):
+ image_url = {"url": part.url}
+ if part.detail is not None:
+ image_url["detail"] = part.detail
+ return {
+ "type": part.type,
+ "image_url": image_url,
+ }
elif isinstance(part, ToolPart):
d = {
"type": part.type,
@@ -168,6 +178,18 @@ class Message:
abstract=p.get("abstract", ""),
)
)
+ elif p["type"] == "image_url":
+ image_url = p.get("image_url")
+ url = ""
+ detail = None
+ if isinstance(image_url, dict):
+ url = str(image_url.get("url", "") or "")
+ detail = image_url.get("detail")
+ elif isinstance(image_url, str):
+ url = image_url
+ if not url.strip():
+ raise ValueError("image_url part requires a non-empty URL")
+ parts.append(ImagePart(url=url, detail=detail))
elif p["type"] == "tool":
parts.append(
ToolPart(
diff --git a/openviking/message/part.py b/openviking/message/part.py
index 36d849368..d738e4836 100644
--- a/openviking/message/part.py
+++ b/openviking/message/part.py
@@ -6,7 +6,7 @@ Message consists of multiple Parts, each Part has different type and purpose.
"""
from dataclasses import dataclass
-from typing import Literal, Optional, Union
+from typing import Any, Dict, Literal, Optional, Union
@dataclass
@@ -30,6 +30,15 @@ class ContextPart:
abstract: str = ""
+@dataclass
+class ImagePart:
+ """Image URL component compatible with OpenAI-style message content."""
+
+ type: Literal["image_url"] = "image_url"
+ url: str = ""
+ detail: Optional[str] = None
+
+
@dataclass
class ToolPart:
"""Tool call component (references tool file within session).
@@ -65,10 +74,19 @@ class ToolPart:
tool_output_group_budget_chars: Optional[int] = None
-Part = Union[TextPart, ContextPart, ToolPart]
+Part = Union[TextPart, ContextPart, ImagePart, ToolPart]
-def part_from_dict(data: dict) -> Part:
+def _parse_image_url_payload(data: Dict[str, Any]) -> tuple[str, Optional[str]]:
+ image_url = data.get("image_url")
+ if isinstance(image_url, dict):
+ return str(image_url.get("url", "") or ""), image_url.get("detail")
+ if isinstance(image_url, str):
+ return image_url, None
+ return "", None
+
+
+def part_from_dict(data: Dict[str, Any]) -> Part:
"""Convert a dict to a Part object.
Args:
@@ -86,6 +104,14 @@ def part_from_dict(data: dict) -> Part:
context_type=data.get("context_type", "memory"),
abstract=data.get("abstract", ""),
)
+ elif part_type == "image_url":
+ url, detail = _parse_image_url_payload(data)
+ if not url.strip():
+ raise ValueError("image_url part requires a non-empty URL")
+ return ImagePart(
+ url=url,
+ detail=detail,
+ )
elif part_type == "tool":
return ToolPart(
tool_id=data.get("tool_id", ""),
diff --git a/openviking/server/routers/sessions.py b/openviking/server/routers/sessions.py
index 3bdf554e4..bc173b98a 100644
--- a/openviking/server/routers/sessions.py
+++ b/openviking/server/routers/sessions.py
@@ -4,7 +4,7 @@
from typing import Any, Dict, List, Literal, Optional
-from fastapi import APIRouter, Body, Depends, Path, Query, Request
+from fastapi import APIRouter, Body, Depends, HTTPException, Path, Query, Request
from pydantic import BaseModel, Field, model_validator
from openviking.core.path_variables import resolve_path_variables
@@ -129,21 +129,37 @@ class CreateSessionRequest(BaseModel):
def _resolve_message_parts(msg_request: AddMessageRequest) -> List[Part]:
"""Resolve parts from an AddMessageRequest, handling path variables."""
if msg_request.parts is not None:
- resolved_parts = []
- for p in msg_request.parts:
- part_copy = dict(p)
- if part_copy.get("type") == "context" and "uri" in part_copy:
- part_copy["uri"] = resolve_path_variables(part_copy["uri"])
- if part_copy.get("type") == "tool":
- if "tool_uri" in part_copy:
- part_copy["tool_uri"] = resolve_path_variables(part_copy["tool_uri"])
- if "skill_uri" in part_copy:
- part_copy["skill_uri"] = resolve_path_variables(part_copy["skill_uri"])
- resolved_parts.append(part_copy)
- return [part_from_dict(p) for p in resolved_parts]
+ return [_part_request_to_part(p) for p in msg_request.parts]
return [TextPart(text=msg_request.content or "")]
+def _part_request_to_part(raw_part: Dict[str, Any]) -> Part:
+ """Convert request part payload into an internal Part."""
+ if not isinstance(raw_part, dict):
+ return TextPart(text=str(raw_part))
+
+ part_copy = dict(raw_part)
+ if part_copy.get("type") == "context" and "uri" in part_copy:
+ part_copy["uri"] = resolve_path_variables(part_copy["uri"])
+ if part_copy.get("type") == "tool":
+ if "tool_uri" in part_copy:
+ part_copy["tool_uri"] = resolve_path_variables(part_copy["tool_uri"])
+ if "skill_uri" in part_copy:
+ part_copy["skill_uri"] = resolve_path_variables(part_copy["skill_uri"])
+ if part_copy.get("type") == "image_url":
+ image_url = part_copy.get("image_url")
+ if isinstance(image_url, dict) and "url" in image_url:
+ image_url = dict(image_url)
+ image_url["url"] = resolve_path_variables(image_url["url"])
+ part_copy["image_url"] = image_url
+ elif isinstance(image_url, str):
+ part_copy["image_url"] = resolve_path_variables(image_url)
+ try:
+ return part_from_dict(part_copy)
+ except ValueError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+
def _to_jsonable(value: Any) -> Any:
"""Convert internal objects (e.g. Context) into JSON-serializable values."""
to_dict = getattr(value, "to_dict", None)
diff --git a/openviking/session/compressor_v2.py b/openviking/session/compressor_v2.py
index fb55d20fb..9ba691b22 100644
--- a/openviking/session/compressor_v2.py
+++ b/openviking/session/compressor_v2.py
@@ -309,6 +309,7 @@ class SessionCompressorV2:
viking_fs=viking_fs,
transaction_handle=transaction_handle,
)
+ await context_provider.prepare_extraction_messages()
extract_context = context_provider.get_extract_context()
isolation_handler = MemoryIsolationHandler(
ctx,
@@ -717,6 +718,8 @@ class SessionCompressorV2:
# Use the provider's extraction context so prompt ranges and memory
# rendering resolve against the same message list.
+ await provider.prepare_extraction_messages()
+
extract_context = provider.get_extract_context()
isolation_handler = MemoryIsolationHandler(
ctx,
diff --git a/openviking/session/memory/core.py b/openviking/session/memory/core.py
index ba22734bd..77b89599b 100644
--- a/openviking/session/memory/core.py
+++ b/openviking/session/memory/core.py
@@ -27,6 +27,12 @@ class ExtractContextProvider(ABC):
"""
pass
+ async def prepare_extraction_messages(self) -> None:
+ """
+ 在构建 prompt、ranges 和 ExtractContext 之前准备 extraction-only messages。
+ """
+ return None
+
@abstractmethod
async def prefetch(
self,
diff --git a/openviking/session/memory/session_extract_context_provider.py b/openviking/session/memory/session_extract_context_provider.py
index 3afed47fe..fb464af82 100644
--- a/openviking/session/memory/session_extract_context_provider.py
+++ b/openviking/session/memory/session_extract_context_provider.py
@@ -10,7 +10,7 @@ import json
import os
from typing import TYPE_CHECKING, Any, Dict, List, Optional
-from openviking.message.part import ToolPart
+from openviking.message.part import TextPart, ToolPart
from openviking.prompts.manager import PromptManager
from openviking.server.identity import RequestContext, ToolContext
from openviking.session.memory.core import ExtractContextProvider
@@ -29,6 +29,9 @@ from openviking.session.memory.tools import (
get_tool,
)
from openviking.session.memory.utils.uri import render_template
+from openviking.session.memory.vision_message_normalizer import (
+ replace_image_parts_with_descriptions,
+)
from openviking.storage.viking_fs import VikingFS
from openviking.telemetry import tracer
from openviking.utils.time_utils import parse_iso_datetime
@@ -58,7 +61,7 @@ class SessionExtractContextProvider(ExtractContextProvider):
viking_fs: VikingFS = None,
transaction_handle=None,
):
- self.messages = messages
+ self.messages = list(messages) if isinstance(messages, list) else messages
self.latest_archive_overview = latest_archive_overview
self._output_language = self._detect_language()
self._registry = None # 延迟加载
@@ -74,6 +77,8 @@ class SessionExtractContextProvider(ExtractContextProvider):
self._viking_fs = viking_fs
self._transaction_handle = transaction_handle
self._link_enabled = config.memory.link_enabled if config.memory else False
+ self._vision_messages_prepared = False
+ self._vision_vlm = None
@property
def read_file_contents(self) -> Dict[str, MemoryFile]:
@@ -104,9 +109,31 @@ class SessionExtractContextProvider(ExtractContextProvider):
)
return self._extract_context
+ async def prepare_extraction_messages(self) -> None:
+ """Prepare extraction-only messages before ranges and prompts are built."""
+ if self._vision_messages_prepared:
+ return
+ if isinstance(self.messages, list):
+ self.messages = await replace_image_parts_with_descriptions(
+ self.messages,
+ get_vlm=self._get_vision_vlm,
+ logger=logger,
+ )
+ self._extract_context = None
+ self._output_language = self._detect_language()
+ self._vision_messages_prepared = True
+
+ def _get_vision_vlm(self):
+ if self._vision_vlm is not None:
+ return self._vision_vlm
+ vlm_config = get_openviking_config().vlm
+ if not (vlm_config and vlm_config.is_available()):
+ return None
+ self._vision_vlm = vlm_config.get_vlm_instance()
+ return self._vision_vlm
+
def _detect_language(self) -> str:
"""检测输出语言"""
- from openviking.message.part import TextPart
from openviking.session.memory.utils import resolve_output_language
user_text_parts = []
diff --git a/openviking/session/memory/vision_message_normalizer.py b/openviking/session/memory/vision_message_normalizer.py
new file mode 100644
index 000000000..3cb73c1a0
--- /dev/null
+++ b/openviking/session/memory/vision_message_normalizer.py
@@ -0,0 +1,108 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Normalize image messages into extraction-friendly text messages."""
+
+from collections.abc import Callable
+from typing import Any, Dict, List
+
+from openviking.message import Message
+from openviking.message.part import ImagePart, TextPart
+
+IMAGE_DESCRIPTION_PROMPT = (
+ "Describe this image for later memory extraction. Focus on durable, user-relevant "
+ "details such as visible people, objects, places, actions, text, dates, and other "
+ "facts that may matter in future conversations. Return only the description."
+)
+
+
+def message_has_image_part(message: Message) -> bool:
+ return any(isinstance(part, ImagePart) for part in getattr(message, "parts", []))
+
+
+def image_part_to_openai_content(part: ImagePart) -> Dict[str, Any]:
+ image_url: Dict[str, Any] = {"url": part.url}
+ if part.detail is not None:
+ image_url["detail"] = part.detail
+ return {"type": "image_url", "image_url": image_url}
+
+
+def build_vision_description_messages(message: Message) -> List[Dict[str, Any]]:
+ content: List[Dict[str, Any]] = [{"type": "text", "text": IMAGE_DESCRIPTION_PROMPT}]
+ for part in getattr(message, "parts", []):
+ if isinstance(part, TextPart) and part.text:
+ content.append({"type": "text", "text": part.text})
+ elif isinstance(part, ImagePart):
+ content.append(image_part_to_openai_content(part))
+ return [{"role": "user", "content": content}]
+
+
+def fallback_image_description(message: Message) -> str:
+ return ""
+
+
+def normalize_vision_response(response: Any) -> str:
+ content = getattr(response, "content", response)
+ return str(content or "").strip()
+
+
+def _original_text_parts(message: Message) -> List[TextPart]:
+ return [
+ TextPart(text=part.text)
+ for part in getattr(message, "parts", [])
+ if isinstance(part, TextPart) and part.text
+ ]
+
+
+async def describe_image_message(
+ message: Message,
+ *,
+ vlm: Any,
+ logger: Any = None,
+) -> str:
+ if vlm is None:
+ return fallback_image_description(message)
+
+ try:
+ response = await vlm.get_vision_completion_async(
+ messages=build_vision_description_messages(message),
+ thinking=False,
+ )
+ description = normalize_vision_response(response)
+ return description or fallback_image_description(message)
+ except Exception as exc:
+ if logger is not None:
+ logger.warning("Failed to describe image message %s: %s", message.id, exc)
+ return fallback_image_description(message)
+
+
+async def replace_image_parts_with_descriptions(
+ messages: List[Message],
+ *,
+ get_vlm: Callable[[], Any],
+ logger: Any = None,
+) -> List[Message]:
+ prepared_messages: List[Message] = []
+ for message in messages:
+ if not message_has_image_part(message):
+ prepared_messages.append(message)
+ continue
+
+ description = await describe_image_message(
+ message,
+ vlm=get_vlm(),
+ logger=logger,
+ )
+ parts = _original_text_parts(message)
+ if description:
+ parts.append(TextPart(text=f"[Image description]: {description}"))
+ if parts:
+ prepared_messages.append(
+ Message(
+ id=message.id,
+ role=message.role,
+ parts=parts,
+ peer_id=message.peer_id,
+ created_at=message.created_at,
+ )
+ )
+ return prepared_messages
diff --git a/openviking/sync_client.py b/openviking/sync_client.py
index 55ee66d4d..80017c4d7 100644
--- a/openviking/sync_client.py
+++ b/openviking/sync_client.py
@@ -107,7 +107,7 @@ class SyncOpenViking:
session_id: Session ID
role: Message role ("user" or "assistant")
content: Text content (simple mode)
- parts: Parts array (full Part support: TextPart, ContextPart, ToolPart)
+ parts: Parts array (full Part support: TextPart, ContextPart, ImagePart, ToolPart)
created_at: Message creation time (ISO format string). If not provided, current time is used.
peer_id: Optional stable interaction peer identity.
diff --git a/openviking_cli/client/base.py b/openviking_cli/client/base.py
index f07b15b3e..47c8a2256 100644
--- a/openviking_cli/client/base.py
+++ b/openviking_cli/client/base.py
@@ -304,7 +304,7 @@ class BaseClient(ABC):
session_id: Session ID
role: Message role ("user" or "assistant")
content: Text content (simple mode)
- parts: Parts array (full Part support: TextPart, ContextPart, ToolPart)
+ parts: Parts array (full Part support: TextPart, ContextPart, ImagePart, ToolPart)
created_at: Message creation time (ISO format string)
peer_id: Optional stable interaction peer identity.
telemetry: Whether to attach operation telemetry data to the result.
diff --git a/openviking_cli/client/sync_http.py b/openviking_cli/client/sync_http.py
index f66a59b1d..474287bcb 100644
--- a/openviking_cli/client/sync_http.py
+++ b/openviking_cli/client/sync_http.py
@@ -143,7 +143,7 @@ class SyncHTTPClient:
session_id: Session ID
role: Message role ("user" or "assistant")
content: Text content (simple mode)
- parts: Parts array (full Part support: TextPart, ContextPart, ToolPart)
+ parts: Parts array (full Part support: TextPart, ContextPart, ImagePart, ToolPart)
created_at: Message creation time (ISO format string)
peer_id: Optional stable interaction peer identity.
diff --git a/tests/client/test_rebuild_clients.py b/tests/client/test_rebuild_clients.py
index df78bdc1d..3d7f153ba 100644
--- a/tests/client/test_rebuild_clients.py
+++ b/tests/client/test_rebuild_clients.py
@@ -11,6 +11,7 @@ import openviking_cli.client.http as http_module
import openviking_cli.utils.async_utils as async_utils
from openviking import AsyncOpenViking, SyncOpenViking
from openviking.client.local import LocalClient
+from openviking.message import ImagePart, TextPart
from openviking_cli.client.http import AsyncHTTPClient
from openviking_cli.client.sync_http import SyncHTTPClient
from openviking_cli.utils.config import OPENVIKING_CLI_CONFIG_ENV
@@ -180,6 +181,50 @@ async def test_local_client_batch_add_messages_forwards_to_session():
assert fake_session.messages[1]["parts"][0].text == "hi"
+async def test_local_client_add_message_accepts_image_parts():
+ class FakeSession:
+ def __init__(self):
+ self.messages = []
+
+ def add_message(self, role, parts, peer_id=None, created_at=None):
+ self.messages.append(
+ {
+ "role": role,
+ "parts": parts,
+ "peer_id": peer_id,
+ "created_at": created_at,
+ }
+ )
+
+ fake_session = FakeSession()
+
+ class FakeSessions:
+ async def get(self, session_id, ctx, auto_create=False):
+ assert session_id == "image-session"
+ assert ctx is client._ctx
+ assert auto_create is True
+ return fake_session
+
+ client = LocalClient.__new__(LocalClient)
+ client._service = SimpleNamespace(sessions=FakeSessions())
+ client._ctx = SimpleNamespace(user=SimpleNamespace(user_id="user-1"))
+
+ result = await LocalClient.add_message(
+ client,
+ "image-session",
+ "user",
+ parts=[
+ {"type": "text", "text": "Look at this"},
+ {"type": "image_url", "image_url": {"url": "https://example.com/image.png"}},
+ ],
+ )
+
+ assert result == {"session_id": "image-session", "message_count": 1}
+ assert isinstance(fake_session.messages[0]["parts"][0], TextPart)
+ assert isinstance(fake_session.messages[0]["parts"][1], ImagePart)
+ assert fake_session.messages[0]["parts"][1].url == "https://example.com/image.png"
+
+
async def test_async_http_client_batch_add_messages_posts_batch_payload():
client = AsyncHTTPClient(url="http://localhost:1933")
fake_http = SimpleNamespace(post=AsyncMock(return_value=object()))
diff --git a/tests/server/test_api_sessions.py b/tests/server/test_api_sessions.py
index da9291491..628ad74cd 100644
--- a/tests/server/test_api_sessions.py
+++ b/tests/server/test_api_sessions.py
@@ -12,7 +12,7 @@ import pytest
from fastapi import FastAPI
from starlette.requests import Request
-from openviking.message import Message, TextPart
+from openviking.message import ImagePart, Message, TextPart
from openviking.server.app import create_app
from openviking.server.config import ServerConfig, ToolOutputExternalizationConfig
from openviking.server.dependencies import set_service
@@ -367,6 +367,169 @@ async def test_add_message(client: httpx.AsyncClient):
assert body["result"]["message_count"] == 1
+async def test_add_message_accepts_image_part(client: httpx.AsyncClient, service):
+ create_resp = await client.post("/api/v1/sessions", json={})
+ session_id = create_resp.json()["result"]["session_id"]
+
+ resp = await client.post(
+ f"/api/v1/sessions/{session_id}/messages",
+ json=_message_request(
+ "user",
+ parts=[
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/image.png",
+ "detail": "auto",
+ },
+ }
+ ],
+ ),
+ )
+
+ assert resp.status_code == 200
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+ session = service.sessions.session(ctx, session_id)
+ await session.load()
+ assert isinstance(session.messages[0].parts[0], ImagePart)
+ assert session.messages[0].parts[0].url == "https://example.com/image.png"
+ assert session.messages[0].parts[0].detail == "auto"
+
+ context_resp = await client.get(f"/api/v1/sessions/{session_id}/context")
+ assert context_resp.status_code == 200
+ assert context_resp.json()["result"]["messages"][0]["parts"][0] == {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/image.png",
+ "detail": "auto",
+ },
+ }
+
+
+async def test_add_message_resolves_image_part_url_path_variables(
+ client: httpx.AsyncClient,
+ service,
+ monkeypatch,
+):
+ monkeypatch.setattr(
+ "openviking.server.routers.sessions.resolve_path_variables",
+ lambda value: value.replace("{calendar:today}", "2026/06/15"),
+ )
+ create_resp = await client.post("/api/v1/sessions", json={})
+ session_id = create_resp.json()["result"]["session_id"]
+
+ resp = await client.post(
+ f"/api/v1/sessions/{session_id}/messages",
+ json=_message_request(
+ "user",
+ parts=[
+ {
+ "type": "image_url",
+ "image_url": {"url": "viking://resources/images/{calendar:today}/photo.png"},
+ }
+ ],
+ ),
+ )
+
+ assert resp.status_code == 200
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+ session = service.sessions.session(ctx, session_id)
+ await session.load()
+ assert isinstance(session.messages[0].parts[0], ImagePart)
+ assert session.messages[0].parts[0].url == "viking://resources/images/2026/06/15/photo.png"
+
+
+async def test_add_message_accepts_mixed_parts(client: httpx.AsyncClient, service):
+ create_resp = await client.post("/api/v1/sessions", json={})
+ session_id = create_resp.json()["result"]["session_id"]
+
+ resp = await client.post(
+ f"/api/v1/sessions/{session_id}/messages",
+ json=_message_request(
+ "user",
+ parts=[
+ {"type": "text", "text": "Look at this"},
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png"},
+ },
+ ],
+ ),
+ )
+
+ assert resp.status_code == 200
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+ session = service.sessions.session(ctx, session_id)
+ await session.load()
+ assert isinstance(session.messages[0].parts[0], TextPart)
+ assert session.messages[0].parts[0].text == "Look at this"
+ assert isinstance(session.messages[0].parts[1], ImagePart)
+ assert session.messages[0].parts[1].url == "https://example.com/image.png"
+
+
+async def test_add_message_rejects_image_part_without_url(client: httpx.AsyncClient):
+ create_resp = await client.post("/api/v1/sessions", json={})
+ session_id = create_resp.json()["result"]["session_id"]
+
+ resp = await client.post(
+ f"/api/v1/sessions/{session_id}/messages",
+ json=_message_request(
+ "user",
+ parts=[{"type": "image_url", "image_url": {}}],
+ ),
+ )
+
+ assert resp.status_code == 400
+ assert "image_url part requires a non-empty URL" in resp.text
+
+
+async def test_add_message_rejects_openai_style_image_content(client: httpx.AsyncClient):
+ create_resp = await client.post("/api/v1/sessions", json={})
+ session_id = create_resp.json()["result"]["session_id"]
+
+ resp = await client.post(
+ f"/api/v1/sessions/{session_id}/messages",
+ json={
+ "role": "user",
+ "content": {
+ "type": "image_url",
+ },
+ },
+ )
+
+ assert resp.status_code == 400
+
+
+async def test_batch_add_message_accepts_mixed_parts(client: httpx.AsyncClient, service):
+ create_resp = await client.post("/api/v1/sessions", json={})
+ session_id = create_resp.json()["result"]["session_id"]
+
+ resp = await client.post(
+ f"/api/v1/sessions/{session_id}/messages/batch",
+ json={
+ "messages": [
+ _message_request(
+ "user",
+ parts=[
+ {"type": "text", "text": "Look at this"},
+ {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png"},
+ },
+ ],
+ )
+ ]
+ },
+ )
+
+ assert resp.status_code == 200
+ ctx = RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+ session = service.sessions.session(ctx, session_id)
+ await session.load()
+ assert isinstance(session.messages[0].parts[0], TextPart)
+ assert isinstance(session.messages[0].parts[1], ImagePart)
+
+
async def test_add_message_splits_tool_result_aggregate(client: httpx.AsyncClient):
create_resp = await client.post("/api/v1/sessions", json={})
session_id = create_resp.json()["result"]["session_id"]
diff --git a/tests/session/memory/test_compressor_v2.py b/tests/session/memory/test_compressor_v2.py
index 8dec9e1f6..79143a807 100644
--- a/tests/session/memory/test_compressor_v2.py
+++ b/tests/session/memory/test_compressor_v2.py
@@ -622,6 +622,9 @@ class TestCompressorV2:
return uri
class DummyProvider:
+ async def prepare_extraction_messages(self):
+ pass
+
def get_memory_schemas(self, _ctx):
return []
@@ -828,6 +831,9 @@ class TestExtractLoopPatchRepair:
def __init__(self):
self.extract_context = ExtractContext([])
+ async def prepare_extraction_messages(self):
+ pass
+
def get_memory_schemas(self, _ctx):
return [schema]
@@ -925,6 +931,9 @@ class TestExtractLoopPatchRepair:
def __init__(self):
self.extract_context = ExtractContext([])
+ async def prepare_extraction_messages(self):
+ pass
+
def get_memory_schemas(self, _ctx):
return [schema]
@@ -1021,6 +1030,9 @@ class TestExtractLoopPatchRepair:
def __init__(self):
self.extract_context = ExtractContext([])
+ async def prepare_extraction_messages(self):
+ pass
+
def get_memory_schemas(self, _ctx):
return [schema]
diff --git a/tests/session/memory/test_memory_react_system_prompt.py b/tests/session/memory/test_memory_react_system_prompt.py
index 61bf63eed..e14f5e274 100644
--- a/tests/session/memory/test_memory_react_system_prompt.py
+++ b/tests/session/memory/test_memory_react_system_prompt.py
@@ -4,8 +4,9 @@
Test that provider instruction correctly instructs LLM.
"""
-from openviking.message import Message, TextPart, ToolPart
+from openviking.message import ImagePart, Message, TextPart, ToolPart
from openviking.session.memory.session_extract_context_provider import SessionExtractContextProvider
+from openviking.session.memory.vision_message_normalizer import IMAGE_DESCRIPTION_PROMPT
class TestProviderInstruction:
@@ -165,3 +166,110 @@ class TestSkillToolCallExposure:
provider = SessionExtractContextProvider(messages=messages)
assert provider._detect_language() == "zh-CN"
+
+ async def test_prepare_extraction_messages_replaces_image_part_with_vlm_description(self):
+ class FakeVisionVLM:
+ def __init__(self):
+ self.messages = None
+
+ async def get_vision_completion_async(self, **kwargs):
+ self.messages = kwargs.get("messages")
+ return "A high level image description."
+
+ messages = [
+ Message(
+ id="m1",
+ role="user",
+ parts=[
+ TextPart("Please remember this image."),
+ ImagePart(url="https://example.com/image.png", detail="auto"),
+ ],
+ )
+ ]
+ provider = SessionExtractContextProvider(messages=messages)
+ fake_vlm = FakeVisionVLM()
+ provider._vision_vlm = fake_vlm
+
+ await provider.prepare_extraction_messages()
+ prompt_message = provider._build_conversation_message()
+
+ assert "Please remember this image." in prompt_message["content"]
+ assert "A high level image description." in prompt_message["content"]
+ assert "https://example.com/image.png" not in prompt_message["content"]
+ assert fake_vlm.messages == [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": IMAGE_DESCRIPTION_PROMPT,
+ },
+ {"type": "text", "text": "Please remember this image."},
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/image.png",
+ "detail": "auto",
+ },
+ },
+ ],
+ }
+ ]
+
+ async def test_prepare_extraction_messages_skips_image_only_without_description(self):
+ messages = [
+ Message(
+ id="m1",
+ role="user",
+ parts=[ImagePart(url="https://example.com/private-family-photo.png")],
+ )
+ ]
+ provider = SessionExtractContextProvider(messages=messages)
+ provider._vision_vlm = None
+
+ await provider.prepare_extraction_messages()
+ prompt_message = provider._build_conversation_message()
+
+ assert "[Image description]" not in prompt_message["content"]
+ assert "https://example.com/private-family-photo.png" not in prompt_message["content"]
+
+ async def test_prepare_extraction_messages_keeps_text_when_image_is_undescribed(self):
+ messages = [
+ Message(
+ id="m1",
+ role="user",
+ parts=[
+ TextPart("Please remember the surrounding text."),
+ ImagePart(url="https://example.com/private-family-photo.png"),
+ ],
+ )
+ ]
+ provider = SessionExtractContextProvider(messages=messages)
+ provider._vision_vlm = None
+
+ await provider.prepare_extraction_messages()
+ prompt_message = provider._build_conversation_message()
+
+ assert "Please remember the surrounding text." in prompt_message["content"]
+ assert "[Image description]" not in prompt_message["content"]
+ assert "https://example.com/private-family-photo.png" not in prompt_message["content"]
+
+ async def test_prepare_extraction_messages_does_not_replace_caller_message_list(self):
+ messages = [
+ Message(
+ id="m1",
+ role="user",
+ parts=[
+ TextPart("Please remember this image."),
+ ImagePart(url="https://example.com/image.png"),
+ ],
+ )
+ ]
+ provider = SessionExtractContextProvider(messages=messages)
+ provider._vision_vlm = None
+
+ await provider.prepare_extraction_messages()
+
+ assert len(messages) == 1
+ assert any(isinstance(part, ImagePart) for part in messages[0].parts)
+ assert provider.messages is not messages
diff --git a/tests/unit/test_message.py b/tests/unit/test_message.py
index 30d803c1f..987bd4f2a 100644
--- a/tests/unit/test_message.py
+++ b/tests/unit/test_message.py
@@ -6,7 +6,9 @@
import json
from datetime import datetime, timezone
-from openviking.message import ContextPart, Message, TextPart, ToolPart
+import pytest
+
+from openviking.message import ContextPart, ImagePart, Message, TextPart, ToolPart
from openviking.message.part import part_from_dict
@@ -108,6 +110,29 @@ class TestContextPart:
assert part.context_type == "resource"
+class TestImagePart:
+ """Test ImagePart dataclass."""
+
+ def test_default_values(self):
+ """Test default values."""
+ part = ImagePart()
+
+ assert part.url == ""
+ assert part.detail is None
+ assert part.type == "image_url"
+
+ def test_custom_values(self):
+ """Test custom values."""
+ part = ImagePart(
+ url="https://example.com/image.png",
+ detail="auto",
+ )
+
+ assert part.url == "https://example.com/image.png"
+ assert part.detail == "auto"
+ assert part.type == "image_url"
+
+
class TestToolPart:
"""Test ToolPart dataclass."""
@@ -258,6 +283,37 @@ class TestPartFromDict:
assert part.tool_output_preview_chars == 100
assert part.tool_output_sha256 == "abc123"
+ def test_image_part_from_openai_style_dict(self):
+ """Test creating ImagePart from OpenAI-style image_url dict."""
+ data = {
+ "type": "image_url",
+ "image_url": {"url": "https://example.com/image.png", "detail": "high"},
+ }
+
+ part = part_from_dict(data)
+
+ assert isinstance(part, ImagePart)
+ assert part.url == "https://example.com/image.png"
+ assert part.detail == "high"
+
+ def test_image_part_rejects_flat_dict(self):
+ """OpenAI-style image_url payloads require the nested image_url shape."""
+ data = {"type": "image_url", "url": "https://example.com/image.png"}
+
+ with pytest.raises(ValueError, match="image_url part requires a non-empty URL"):
+ part_from_dict(data)
+
+ def test_image_part_rejects_missing_url(self):
+ """Test image_url parts require a non-empty URL."""
+ data = {"type": "image_url", "image_url": {}}
+
+ try:
+ part_from_dict(data)
+ except ValueError as exc:
+ assert "image_url part requires a non-empty URL" in str(exc)
+ else:
+ raise AssertionError("Expected ValueError for missing image URL")
+
def test_unknown_type_defaults_to_text(self):
"""Test unknown type defaults to TextPart."""
data = {"type": "unknown", "value": "something"}
@@ -473,6 +529,30 @@ class TestMessageToDict:
assert d["parts"][0]["tool_output_preview_chars"] == 100
assert d["parts"][0]["tool_output_externalized_reason"] == "single_threshold"
+ def test_to_dict_with_image_part(self):
+ """Test to_dict with ImagePart."""
+ msg = Message(
+ id="msg-1",
+ role="user",
+ parts=[
+ TextPart(text="Look at this"),
+ ImagePart(
+ url="https://example.com/image.png",
+ detail="auto",
+ ),
+ ],
+ )
+
+ d = msg.to_dict()
+
+ assert d["parts"][1] == {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/image.png",
+ "detail": "auto",
+ },
+ }
+
def test_to_dict_timestamp_format(self):
"""Test timestamp format in to_dict."""
now = datetime(2026, 3, 26, 10, 30, 0, tzinfo=timezone.utc)
@@ -561,6 +641,31 @@ class TestMessageFromDict:
assert msg.parts[0].tool_output_preview_chars == 100
assert msg.parts[0].tool_output_externalized_reason == "single_threshold"
+ def test_from_dict_with_image_part(self):
+ """Test from_dict with ImagePart."""
+ d = {
+ "id": "msg-1",
+ "role": "user",
+ "parts": [
+ {"type": "text", "text": "Look at this"},
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://example.com/image.png",
+ "detail": "auto",
+ },
+ },
+ ],
+ "created_at": "2026-03-26T10:30:00Z",
+ }
+
+ msg = Message.from_dict(d)
+
+ assert isinstance(msg.parts[0], TextPart)
+ assert isinstance(msg.parts[1], ImagePart)
+ assert msg.parts[1].url == "https://example.com/image.png"
+ assert msg.parts[1].detail == "auto"
+
def test_from_dict_supports_legacy_content_only_messages(self):
"""Legacy messages with only content should load as a TextPart."""
d = {