Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(bot): support multimodal OpenViking resource reads (#4593)

DuTao6020c62ccc2c
diff --git a/bot/tests/test_agent_loop_outcome.py b/bot/tests/test_agent_loop_outcome.py
index da49eed79..7b8f17806 100644
--- a/bot/tests/test_agent_loop_outcome.py
+++ b/bot/tests/test_agent_loop_outcome.py
@@ -1,3 +1,4 @@
+import copy
import sys
from datetime import datetime
from pathlib import Path
@@ -9,6 +10,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from vikingbot.agent import loop as loop_module
from vikingbot.agent.context import ContextBuilder
from vikingbot.agent.loop import AgentLoop
+from vikingbot.agent.tools.base import MultimodalToolResult
+from vikingbot.agent.tools.registry import ToolExecutionResult
from vikingbot.bus.events import InboundMessage, OutboundEventType
from vikingbot.bus.queue import MessageBus
from vikingbot.config.schema import AgentsConfig, Config, SessionKey
@@ -108,6 +111,305 @@ def test_agents_config_keeps_ten_recent_openviking_messages_by_default():
assert AgentsConfig().commit_keep_recent_count == 10
+def test_context_keeps_multimodal_tool_result_on_tool_message(temp_dir: Path):
+ context = ContextBuilder(workspace=temp_dir / "workspace")
+ content = [
+ {"type": "text", "text": "Source: viking://resources/image.png"},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,aW1hZ2U="},
+ },
+ ]
+ messages = context.add_tool_result(
+ [],
+ "call-1",
+ "openviking_multi_read",
+ MultimodalToolResult(text="Image resource.", content=content),
+ )
+
+ assert messages == [
+ {
+ "role": "tool",
+ "tool_call_id": "call-1",
+ "name": "openviking_multi_read",
+ "content": content,
+ }
+ ]
+
+
[email protected]("supports_media", [False, True])
+async def test_agent_loop_gates_multimodal_result_by_provider(
+ temp_dir: Path, monkeypatch, supports_media: bool
+):
+ monkeypatch.setattr(AgentLoop, "_register_builtin_hooks", lambda self: None)
+ monkeypatch.setattr(AgentLoop, "_register_default_tools", lambda self: None)
+ monkeypatch.setattr("vikingbot.agent.loop.SubagentManager", _FakeSubagentManager)
+
+ content = [
+ {"type": "text", "text": "Source: viking://resources/image.png"},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,aW1hZ2U="},
+ },
+ ]
+
+ class Provider(LLMProvider):
+ def __init__(self):
+ super().__init__()
+ self.calls = []
+
+ async def chat(self, messages, tools=None, **kwargs):
+ self.calls.append(messages)
+ if len(self.calls) == 1:
+ return LLMResponse(
+ content=None,
+ tool_calls=[
+ ToolCallRequest(
+ id="call-1",
+ name="read_image",
+ arguments={},
+ tokens=1,
+ )
+ ],
+ )
+ return LLMResponse(content="done")
+
+ def get_default_model(self) -> str:
+ return "fake-model"
+
+ def supports_tool_result_media(self, model=None) -> bool:
+ return supports_media
+
+ class Registry:
+ def get_definitions(self, **kwargs):
+ return [
+ {
+ "type": "function",
+ "function": {
+ "name": "read_image",
+ "description": "Read an image",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ]
+
+ async def execute_detailed(self, name, params, **kwargs):
+ return ToolExecutionResult(
+ result=MultimodalToolResult(text="Image resource.", content=content),
+ effective_params=params,
+ )
+
+ provider = Provider()
+ loop = AgentLoop(
+ bus=MessageBus(),
+ provider=provider,
+ workspace=temp_dir / "workspace",
+ config=Config(storage_workspace=str(temp_dir)),
+ max_iterations=2,
+ )
+
+ final, _reasoning, tools_used, _usage, _iteration = await loop._run_agent_loop(
+ messages=[{"role": "user", "content": "read it"}],
+ session_key=SessionKey(type="cli", channel_id="default", chat_id="multimodal"),
+ publish_events=False,
+ tool_registry=Registry(),
+ )
+
+ assert final == "done"
+ tool_message = next(message for message in provider.calls[1] if message["role"] == "tool")
+ if supports_media:
+ assert tool_message["content"] == content
+ else:
+ assert tool_message["content"].startswith("Image resource.")
+ assert "does not support media in tool results" in tool_message["content"]
+ assert tools_used[0]["result"] == "Image resource."
+
+
+async def test_agent_loop_limits_media_across_parallel_tool_results(temp_dir: Path, monkeypatch):
+ monkeypatch.setattr(AgentLoop, "_register_builtin_hooks", lambda self: None)
+ monkeypatch.setattr(AgentLoop, "_register_default_tools", lambda self: None)
+ monkeypatch.setattr("vikingbot.agent.loop.SubagentManager", _FakeSubagentManager)
+ monkeypatch.setattr(loop_module, "MAX_INLINE_TOOL_RESULT_MEDIA_BYTES", 5)
+
+ class Provider(LLMProvider):
+ def __init__(self):
+ super().__init__()
+ self.calls = []
+
+ async def chat(self, messages, tools=None, **kwargs):
+ self.calls.append(messages)
+ if len(self.calls) == 1:
+ return LLMResponse(
+ content=None,
+ tool_calls=[
+ ToolCallRequest(
+ id="call-1",
+ name="read_image",
+ arguments={"label": "first"},
+ tokens=1,
+ ),
+ ToolCallRequest(
+ id="call-2",
+ name="read_image",
+ arguments={"label": "second"},
+ tokens=1,
+ ),
+ ],
+ )
+ return LLMResponse(content="done")
+
+ def get_default_model(self) -> str:
+ return "fake-model"
+
+ def supports_tool_result_media(self, model=None) -> bool:
+ return True
+
+ class Registry:
+ def get_definitions(self, **kwargs):
+ return [
+ {
+ "type": "function",
+ "function": {
+ "name": "read_image",
+ "description": "Read an image",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ]
+
+ async def execute_detailed(self, name, params, **kwargs):
+ label = params["label"]
+ return ToolExecutionResult(
+ result=MultimodalToolResult(
+ text=f"{label} image",
+ content=[
+ {"type": "text", "text": label},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,ZGF0YQ=="},
+ },
+ ],
+ ),
+ effective_params=params,
+ )
+
+ provider = Provider()
+ loop = AgentLoop(
+ bus=MessageBus(),
+ provider=provider,
+ workspace=temp_dir / "workspace",
+ config=Config(storage_workspace=str(temp_dir)),
+ max_iterations=2,
+ )
+
+ final, *_ = await loop._run_agent_loop(
+ messages=[{"role": "user", "content": "read both"}],
+ session_key=SessionKey(type="cli", channel_id="default", chat_id="media-budget"),
+ publish_events=False,
+ tool_registry=Registry(),
+ )
+
+ assert final == "done"
+ tool_messages = [message for message in provider.calls[1] if message["role"] == "tool"]
+ assert isinstance(tool_messages[0]["content"], list)
+ assert isinstance(tool_messages[1]["content"], str)
+ assert "make this model request exceed" in tool_messages[1]["content"]
+
+
+async def test_agent_loop_prefers_new_media_across_consecutive_tool_rounds(
+ temp_dir: Path, monkeypatch
+):
+ monkeypatch.setattr(AgentLoop, "_register_builtin_hooks", lambda self: None)
+ monkeypatch.setattr(AgentLoop, "_register_default_tools", lambda self: None)
+ monkeypatch.setattr("vikingbot.agent.loop.SubagentManager", _FakeSubagentManager)
+ monkeypatch.setattr(loop_module, "MAX_INLINE_TOOL_RESULT_MEDIA_BYTES", 5)
+
+ class Provider(LLMProvider):
+ def __init__(self):
+ super().__init__()
+ self.calls = []
+
+ async def chat(self, messages, tools=None, **kwargs):
+ self.calls.append(copy.deepcopy(messages))
+ call_number = len(self.calls)
+ if call_number <= 2:
+ return LLMResponse(
+ content=None,
+ tool_calls=[
+ ToolCallRequest(
+ id=f"call-{call_number}",
+ name="read_image",
+ arguments={"label": f"image-{call_number}"},
+ tokens=1,
+ )
+ ],
+ )
+ return LLMResponse(content="done")
+
+ def get_default_model(self) -> str:
+ return "fake-model"
+
+ def supports_tool_result_media(self, model=None) -> bool:
+ return True
+
+ class Registry:
+ def get_definitions(self, **kwargs):
+ return [
+ {
+ "type": "function",
+ "function": {
+ "name": "read_image",
+ "description": "Read an image",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ]
+
+ async def execute_detailed(self, name, params, **kwargs):
+ label = params["label"]
+ return ToolExecutionResult(
+ result=MultimodalToolResult(
+ text=f"{label} result",
+ content=[
+ {"type": "text", "text": label},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,ZGF0YQ=="},
+ },
+ ],
+ ),
+ effective_params=params,
+ )
+
+ provider = Provider()
+ loop = AgentLoop(
+ bus=MessageBus(),
+ provider=provider,
+ workspace=temp_dir / "workspace",
+ config=Config(storage_workspace=str(temp_dir)),
+ max_iterations=3,
+ )
+
+ final, *_ = await loop._run_agent_loop(
+ messages=[{"role": "user", "content": "read two images in sequence"}],
+ session_key=SessionKey(type="cli", channel_id="default", chat_id="media-rounds"),
+ publish_events=False,
+ tool_registry=Registry(),
+ )
+
+ assert final == "done"
+ first_round_tool = next(message for message in provider.calls[1] if message["role"] == "tool")
+ assert isinstance(first_round_tool["content"], list)
+
+ tool_messages = [message for message in provider.calls[-1] if message["role"] == "tool"]
+ assert isinstance(tool_messages[0]["content"], str)
+ assert "earlier tool result was omitted" in tool_messages[0]["content"]
+ assert isinstance(tool_messages[1]["content"], list)
+
+
def test_agent_loop_omits_spawn_tool_when_subagents_disabled(temp_dir: Path, monkeypatch):
monkeypatch.setattr(AgentLoop, "_register_builtin_hooks", lambda self: None)
monkeypatch.setattr("vikingbot.agent.loop.SubagentManager", _FakeSubagentManager)
diff --git a/bot/tests/test_compile.py b/bot/tests/test_compile.py
index 5e424e504..facc5bfa6 100644
--- a/bot/tests/test_compile.py
+++ b/bot/tests/test_compile.py
@@ -11,7 +11,7 @@ from vikingbot.agent.loop import (
AgentLoop,
render_budget_reminder,
)
-from vikingbot.agent.tools.base import Tool, ToolContext
+from vikingbot.agent.tools.base import MultimodalToolResult, Tool, ToolContext
from vikingbot.agent.tools.compile import (
CompileScopedTool,
SubmitTargetCheckoutTool,
@@ -1366,6 +1366,146 @@ async def test_multi_read_hints_on_large_files_and_supports_offset_limit(monkeyp
assert fake.calls == [("viking://resources/big.jsonl", 10, 20)]
+async def test_multi_read_returns_images_as_tool_result_content(monkeypatch):
+ image_uri = "viking://resources/example/image.png"
+ image_bytes = b"\x89PNG\r\n\x1a\nimage-data"
+
+ class FakeClient:
+ async def stat(self, uri):
+ assert uri == image_uri
+ return {"size": len(image_bytes), "isDir": False}
+
+ async def download_bytes(self, uri):
+ assert uri == image_uri
+ return image_bytes
+
+ async def read_content(self, *args, **kwargs):
+ raise AssertionError("image reads must not use the text endpoint")
+
+ tool = VikingMultiReadTool()
+
+ async def fake_get_client(ctx):
+ return FakeClient()
+
+ async def no_release(ctx, client):
+ return None
+
+ monkeypatch.setattr(tool, "_get_client", fake_get_client)
+ monkeypatch.setattr(tool, "_release_client", no_release)
+
+ result = await tool.execute(ToolContext(), uris=[image_uri])
+
+ assert isinstance(result, MultimodalToolResult)
+ assert f"START OF {image_uri}" in str(result)
+ assert "Image resource (image/png" in str(result)
+ assert result.content[2] == {
+ "type": "image_url",
+ "image_url": {
+ "url": f"data:image/png;base64,{base64.b64encode(image_bytes).decode('ascii')}"
+ },
+ }
+
+
+async def test_multi_read_treats_typescript_as_text_and_preserves_window(monkeypatch):
+ typescript_uri = "viking://resources/web/index.ts"
+
+ class FakeClient:
+ async def read_content(self, uri, level="abstract", offset=0, limit=-1, **kwargs):
+ assert uri == typescript_uri
+ assert level == "read"
+ assert (offset, limit) == (4, 2)
+ return "export const answer = 42;"
+
+ async def download_bytes(self, uri):
+ raise AssertionError("TypeScript reads must not use the media download path")
+
+ tool = VikingMultiReadTool()
+
+ async def fake_get_client(ctx):
+ return FakeClient()
+
+ async def no_release(ctx, client):
+ return None
+
+ monkeypatch.setattr(tool, "_get_client", fake_get_client)
+ monkeypatch.setattr(tool, "_release_client", no_release)
+
+ result = await tool.execute(ToolContext(), uris=[typescript_uri], offset=4, limit=2)
+
+ assert isinstance(result, str)
+ assert "export const answer = 42;" in result
+
+
+async def test_multi_read_limits_aggregate_inline_image_bytes(monkeypatch):
+ first_uri = "viking://resources/first.png"
+ second_uri = "viking://resources/second.png"
+ first_image = b"\x89PNG\r\n\x1a\nfirst"
+ second_image = b"\x89PNG\r\n\x1a\nother"
+ images = {first_uri: first_image, second_uri: second_image}
+
+ class FakeClient:
+ async def stat(self, uri):
+ return {"size": len(images[uri]), "isDir": False}
+
+ async def download_bytes(self, uri):
+ return images[uri]
+
+ async def read_content(self, *args, **kwargs):
+ raise AssertionError("image reads must not use the text endpoint")
+
+ tool = VikingMultiReadTool()
+ monkeypatch.setattr(tool, "_MAX_INLINE_MEDIA_BYTES", len(first_image))
+
+ async def fake_get_client(ctx):
+ return FakeClient()
+
+ async def no_release(ctx, client):
+ return None
+
+ monkeypatch.setattr(tool, "_get_client", fake_get_client)
+ monkeypatch.setattr(tool, "_release_client", no_release)
+
+ result = await tool.execute(ToolContext(), uris=[first_uri, second_uri])
+
+ assert isinstance(result, MultimodalToolResult)
+ assert sum(part.get("type") == "image_url" for part in result.content) == 1
+ assert "combined inline media size" in str(result)
+
+
[email protected]("extension", ["mp3", "mp4"])
+async def test_multi_read_uses_openviking_overview_for_audio_and_video(monkeypatch, extension):
+ media_uri = f"viking://resources/interview/interview.{extension}"
+
+ class FakeClient:
+ async def read_content(self, uri, level="abstract", **kwargs):
+ assert uri == media_uri
+ assert level == "overview"
+ return "Speaker: hello"
+
+ async def download_bytes(self, uri):
+ raise AssertionError("media reads must not create a second transcription path")
+
+ tool = VikingMultiReadTool()
+
+ async def fake_get_client(ctx):
+ return FakeClient()
+
+ async def no_release(ctx, client):
+ return None
+
+ monkeypatch.setattr(tool, "_get_client", fake_get_client)
+ monkeypatch.setattr(tool, "_release_client", no_release)
+
+ result = await tool.execute(ToolContext(), uris=[media_uri])
+
+ assert isinstance(result, str)
+ assert "Speaker: hello" in result
+
+
class _RecordingSandbox:
def __init__(self):
self.writes = []
diff --git a/bot/tests/test_image_format.py b/bot/tests/test_image_format.py
index 4dfc27efb..71a3e8337 100644
--- a/bot/tests/test_image_format.py
+++ b/bot/tests/test_image_format.py
@@ -7,7 +7,7 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
-from vikingbot.utils.image_format import detect_image_format # noqa: E402
+from vikingbot.utils.image_format import detect_image_format, sniff_image_format # noqa: E402
def test_detect_image_format_identifies_jpeg():
@@ -22,3 +22,7 @@ def test_detect_image_format_uses_mime_hint_for_unknown_bytes():
assert image_format.extension == "webp"
assert image_format.mime_type == "image/webp"
+
+
+def test_sniff_image_format_rejects_unknown_bytes():
+ assert sniff_image_format(b"not an image") is None
diff --git a/bot/tests/test_openviking_api_key_type.py b/bot/tests/test_openviking_api_key_type.py
index 0ef698403..3d51a3c26 100644
--- a/bot/tests/test_openviking_api_key_type.py
+++ b/bot/tests/test_openviking_api_key_type.py
@@ -48,6 +48,8 @@ class _DummyHTTPClient:
self.find_calls = []
self.ls_calls = []
self.read_calls = []
+ self.stat_calls = []
+ self.download_calls = []
self.closed = False
_DummyHTTPClient.instances.append(self)
@@ -105,6 +107,14 @@ class _DummyHTTPClient:
self.read_calls.append(("read", uri))
return ""
+ async def stat(self, uri):
+ self.stat_calls.append(uri)
+ return {"size": 8, "isDir": False}
+
+ async def download_bytes(self, uri):
+ self.download_calls.append(uri)
+ return b"image"
+
async def grep(self, *_args, **_kwargs):
return {"matches": []}
@@ -1958,6 +1968,27 @@ async def test_read_content_trusted_owner_uri_uses_owner_identity(monkeypatch):
assert scoped.closed is True
+async def test_image_reads_trusted_owner_uri_use_owner_identity(monkeypatch):
+ monkeypatch.setattr(ov_server_module, "load_config", lambda: _make_config("root"))
+ client = VikingClient()
+ uri = "viking://user/sender-1/resources/image.png"
+
+ stat = await client.stat(uri)
+ content = await client.download_bytes(uri)
+
+ assert stat == {"size": 8, "isDir": False}
+ assert content == b"image"
+ stat_client, download_client = _DummyHTTPClient.instances[1:]
+ for scoped in (stat_client, download_client):
+ assert scoped.kwargs["api_key"] == "root-key"
+ assert scoped.kwargs["account"] == "acct"
+ assert scoped.kwargs["user"] == "sender-1"
+ assert scoped.closed is True
+ assert stat_client.stat_calls == [uri]
+ assert download_client.download_calls == [uri]
+
+
@pytest.mark.asyncio
async def test_search_memory_peer_ids_use_explicit_peer_uris(monkeypatch):
monkeypatch.setattr(ov_server_module, "load_config", lambda: _make_config("user"))
diff --git a/bot/vikingbot/agent/context.py b/bot/vikingbot/agent/context.py
index dc188df0f..f317b6153 100644
--- a/bot/vikingbot/agent/context.py
+++ b/bot/vikingbot/agent/context.py
@@ -12,6 +12,7 @@ from loguru import logger
from vikingbot.agent.memory import MemoryStore
from vikingbot.agent.skills import SkillsLoader
+from vikingbot.agent.tools.base import MultimodalToolResult
from vikingbot.config.schema import SessionKey
from vikingbot.sandbox import SandboxManager
from vikingbot.utils.helpers import ensure_non_empty_assistant_content
@@ -510,7 +511,7 @@ IMPORTANT:
return images + [{"type": "text", "text": text}]
def add_tool_result(
- self, messages: list[dict[str, Any]], tool_call_id: str, tool_name: str, result: str
+ self, messages: list[dict[str, Any]], tool_call_id: str, tool_name: str, result: Any
) -> list[dict[str, Any]]:
"""
Add a tool result to the message list.
@@ -524,8 +525,9 @@ IMPORTANT:
Returns:
Updated message list.
"""
+ content = result.content if isinstance(result, MultimodalToolResult) else result
messages.append(
- {"role": "tool", "tool_call_id": tool_call_id, "name": tool_name, "content": result}
+ {"role": "tool", "tool_call_id": tool_call_id, "name": tool_name, "content": content}
)
return messages
diff --git a/bot/vikingbot/agent/loop.py b/bot/vikingbot/agent/loop.py
index 91bc6a73b..1c860c70a 100644
--- a/bot/vikingbot/agent/loop.py
+++ b/bot/vikingbot/agent/loop.py
@@ -15,12 +15,14 @@ from typing import TYPE_CHECKING, Any
from loguru import logger
+from openviking.utils.media_limits import MAX_INLINE_TOOL_RESULT_MEDIA_BYTES
from vikingbot.agent.context import ContextBuilder
from vikingbot.agent.memory import MemoryStore
from vikingbot.agent.remote_skills import SkillRuntimeContext
from vikingbot.agent.skills import SkillsLoader
from vikingbot.agent.subagent import SubagentManager
from vikingbot.agent.tools import register_default_tools
+from vikingbot.agent.tools.base import MultimodalToolResult
from vikingbot.agent.tools.registry import ToolExecutionResult, ToolRegistry
from vikingbot.bus.events import InboundMessage, OutboundEventType, OutboundMessage
from vikingbot.bus.queue import MessageBus
@@ -59,6 +61,75 @@ def _compact_msg_chars(messages: list[dict]) -> int:
return sum(len(json.dumps(m, ensure_ascii=False, default=str)) for m in messages)
+def _base64_data_url_bytes(url: str) -> int:
+ if not url.startswith("data:"):
+ return 0
+ marker = ";base64,"
+ marker_index = url.find(marker)
+ if marker_index < 0:
+ return 0
+ encoded_length = len(url) - marker_index - len(marker)
+ if encoded_length <= 0:
+ return 0
+ padding = 2 if url.endswith("==") else 1 if url.endswith("=") else 0
+ return max(0, encoded_length * 3 // 4 - padding)
+
+
+def _inline_media_bytes(messages: list[dict]) -> int:
+ total = 0
+ for message in messages:
+ content = message.get("content")
+ if not isinstance(content, list):
+ continue
+ for part in content:
+ if not isinstance(part, dict) or part.get("type") not in {
+ "image_url",
+ "input_image",
+ }:
+ continue
+ image_url = part.get("image_url")
+ url = image_url.get("url") if isinstance(image_url, dict) else image_url
+ if isinstance(url, str):
+ total += _base64_data_url_bytes(url)
+ return total
+
+
+def _demote_historical_tool_media(
+ messages: list[dict], *, history_end: int, bytes_needed: int
+) -> int:
+ candidates: list[tuple[dict, int]] = []
+ reclaimable_bytes = 0
+ for message in messages[:history_end]:
+ if message.get("role") != "tool" or not isinstance(message.get("content"), list):
+ continue
+ media_bytes = _inline_media_bytes([message])
+ if media_bytes <= 0:
+ continue
+ candidates.append((message, media_bytes))
+ reclaimable_bytes += media_bytes
+ if reclaimable_bytes >= bytes_needed:
+ break
+
+ if reclaimable_bytes < bytes_needed:
+ return 0
+
+ for message, _media_bytes in candidates:
+ text_parts = [
+ part["text"]
+ for part in message["content"]
+ if isinstance(part, dict)
+ and part.get("type") in {"text", "input_text"}
+ and isinstance(part.get("text"), str)
+ ]
+ text = "\n".join(text_parts)
+ omission_note = (
+ "Media content from this earlier tool result was omitted from subsequent "
+ "requests to keep the inline media budget."
+ )
+ message["content"] = f"{text}\n\n{omission_note}" if text else omission_note
+ return reclaimable_bytes
+
+
def _compact_render_message(message: dict) -> str:
role = message.get("role")
content = message.get("content")
@@ -136,9 +207,9 @@ def _compact_split_chunks(text: str) -> list[str]:
def _compact_strip_note_header(content: str) -> str:
text = str(content or "").strip()
if text.startswith("[Context compaction]"):
- text = text[len("[Context compaction]"):].strip()
+ text = text[len("[Context compaction]") :].strip()
if text.startswith("Findings so far:"):
- text = text[len("Findings so far:"):].strip()
+ text = text[len("Findings so far:") :].strip()
return text.strip()
@@ -1594,21 +1665,59 @@ class AgentLoop:
# Stage 3: Process results sequentially in original order
turn_tools: list[dict[str, Any]] = []
+ history_end = len(messages)
+ request_media_bytes = _inline_media_bytes(messages)
for _idx, tool_call, outcome, tool_execute_duration in results:
result = outcome.result
+ result_text = str(result)
+ recorded_result = (
+ result_text if isinstance(result, MultimodalToolResult) else result
+ )
args_str = json.dumps(tool_call.arguments, ensure_ascii=False)
- logger.info(f"[RESULT]: {str(result)[:600]}")
+ logger.info(f"[RESULT]: {result_text[:600]}")
if publish_events:
await self.bus.publish_outbound(
OutboundMessage(
session_key=session_key,
- content=str(result),
+ content=result_text,
event_type=OutboundEventType.TOOL_RESULT,
)
)
+ model_result = result
+ if isinstance(result, MultimodalToolResult):
+ result_media_bytes = _inline_media_bytes([{"content": result.content}])
+ if not self.provider.supports_tool_result_media(self.model):
+ model_result = (
+ f"{result_text}\n\nMedia content was omitted because the configured "
+ "model provider does not support media in tool results."
+ )
+ else:
+ overflow = (
+ request_media_bytes
+ + result_media_bytes
+ - MAX_INLINE_TOOL_RESULT_MEDIA_BYTES
+ )
+ if overflow > 0:
+ reclaimed = _demote_historical_tool_media(
+ messages,
+ history_end=history_end,
+ bytes_needed=overflow,
+ )
+ request_media_bytes -= reclaimed
+ if (
+ request_media_bytes + result_media_bytes
+ > MAX_INLINE_TOOL_RESULT_MEDIA_BYTES
+ ):
+ model_result = (
+ f"{result_text}\n\nMedia content was omitted because adding it would "
+ "make this model request exceed the inline media "
+ f"limit of {MAX_INLINE_TOOL_RESULT_MEDIA_BYTES} bytes."
+ )
+ else:
+ request_media_bytes += result_media_bytes
messages = self.context.add_tool_result(
- messages, tool_call.id, tool_call.name, result
+ messages, tool_call.id, tool_call.name, model_result
)
tool_used_dict = {
@@ -1616,11 +1725,11 @@ class AgentLoop:
"tool_name": tool_call.name,
"args": args_str,
"resolved_args": outcome.effective_params,
- "result": result,
+ "result": recorded_result,
"duration": tool_execute_duration,
"execute_success": _is_tool_result_success(result),
"input_token": tool_call.tokens,
- "output_token": cal_str_tokens(result, text_type="mixed"),
+ "output_token": cal_str_tokens(result_text, text_type="mixed"),
}
if outcome.skill_uris:
tool_used_dict["skill_uri"] = outcome.skill_uris[0]
diff --git a/bot/vikingbot/agent/tools/base.py b/bot/vikingbot/agent/tools/base.py
index 36a47a8a0..17bc9def6 100644
--- a/bot/vikingbot/agent/tools/base.py
+++ b/bot/vikingbot/agent/tools/base.py
@@ -8,6 +8,17 @@ from vikingbot.config.schema import SessionKey
from vikingbot.sandbox.manager import SandboxManager
+@dataclass(frozen=True)
+class MultimodalToolResult:
+ """A tool result with model-facing content and a text-only representation."""
+
+ text: str
+ content: list[dict[str, Any]]
+
+ def __str__(self) -> str:
+ return self.text
+
+
@dataclass
class ToolContext:
"""Context passed to tools during execution, containing runtime information.
@@ -152,7 +163,7 @@ class Tool(ABC):
return {}
@abstractmethod
- async def execute(self, tool_context: ToolContext, **kwargs: Any) -> str:
+ async def execute(self, tool_context: ToolContext, **kwargs: Any) -> str | MultimodalToolResult:
"""
Execute the tool with given parameters.
@@ -161,7 +172,7 @@ class Tool(ABC):
**kwargs: Tool-specific parameters.
Returns:
- String result of the tool execution.
+ Text result, optionally with model-facing multimodal content.
"""
pass
diff --git a/bot/vikingbot/agent/tools/ov_file.py b/bot/vikingbot/agent/tools/ov_file.py
index 398c7999c..f05d862ab 100644
--- a/bot/vikingbot/agent/tools/ov_file.py
+++ b/bot/vikingbot/agent/tools/ov_file.py
@@ -1,19 +1,22 @@
"""OpenViking file system tools: read, write, list, search resources."""
import asyncio
+import base64
import itertools
import json
import tempfile
import time
from abc import ABC
-from pathlib import Path
+from pathlib import Path, PurePosixPath
from typing import TYPE_CHECKING, Any, Mapping, Optional
import httpx
from loguru import logger
-from vikingbot.agent.tools.base import Tool, ToolContext
+from openviking.utils.media_limits import MAX_INLINE_TOOL_RESULT_MEDIA_BYTES
+from vikingbot.agent.tools.base import MultimodalToolResult, Tool, ToolContext
from vikingbot.openviking_mount.ov_server import VikingClient
+from vikingbot.utils.image_format import sniff_image_format
if TYPE_CHECKING:
from vikingbot.config.schema import Config
@@ -919,6 +922,30 @@ class VikingMultiReadTool(OVFileTool):
"""Tool to read content from multiple Viking resources concurrently."""
_FULL_READ_WARN_BYTES = 512 * 1024
+ _MAX_INLINE_IMAGES = 4
+ _MAX_INLINE_MEDIA_BYTES = MAX_INLINE_TOOL_RESULT_MEDIA_BYTES
+ _INLINE_IMAGE_EXTENSIONS = frozenset({".gif", ".jpeg", ".jpg", ".png", ".webp"})
+ _INLINE_IMAGE_MIME_TYPES = frozenset({"image/gif", "image/jpeg", "image/png", "image/webp"})
+ _AUDIO_EXTENSIONS = frozenset(
+ {".aac", ".ac3", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav"}
+ )
+ _VIDEO_EXTENSIONS = frozenset({".avi", ".flv", ".mkv", ".mov", ".mp4", ".webm", ".wmv"})
+
+ @staticmethod
+ def _uri_suffix(uri: str) -> str:
+ path = uri.split("#", 1)[0].split("?", 1)[0]
+ return PurePosixPath(path).suffix.lower()
+
+ @classmethod
+ def _media_kind(cls, uri: str) -> str | None:
+ suffix = cls._uri_suffix(uri)
+ if suffix in cls._INLINE_IMAGE_EXTENSIONS:
+ return "image"
+ if suffix in cls._AUDIO_EXTENSIONS:
+ return "audio"
+ if suffix in cls._VIDEO_EXTENSIONS:
+ return "video"
+ return None
@property
def name(self) -> str:
@@ -964,7 +991,7 @@ class VikingMultiReadTool(OVFileTool):
offset: int = 0,
limit: int = -1,
**kwargs: Any,
- ) -> str:
+ ) -> str | MultimodalToolResult:
level = "read" # 默认获取完整内容
client = None
try:
@@ -975,9 +1002,104 @@ class VikingMultiReadTool(OVFileTool):
max_concurrent = 10
semaphore = asyncio.Semaphore(max_concurrent)
- async def read_single_uri(uri: str) -> dict:
+ image_indexes = [
+ index for index, uri in enumerate(uris) if self._media_kind(uri) == "image"
+ ]
+ inline_image_indexes = set(image_indexes[: self._MAX_INLINE_IMAGES])
+
+ async def read_single_uri(index: int, uri: str) -> dict[str, Any]:
async with semaphore:
try:
+ media_kind = self._media_kind(uri)
+ if media_kind == "image":
+ if index not in inline_image_indexes:
+ return {
+ "uri": uri,
+ "content": (
+ "Image was not attached because one tool result supports "
+ f"at most {self._MAX_INLINE_IMAGES} images. Read fewer "
+ "images at once."
+ ),
+ "success": False,
+ "media_kind": media_kind,
+ }
+ stat = await client.stat(uri)
+ size = stat.get("size") if isinstance(stat, dict) else None
+ if isinstance(size, bool) or not isinstance(size, int) or size < 0:
+ return {
+ "uri": uri,
+ "content": "Image size is unavailable; cannot attach it safely.",
+ "success": False,
+ "media_kind": media_kind,
+ }
+ if size > self._MAX_INLINE_MEDIA_BYTES:
+ return {
+ "uri": uri,
+ "content": (
+ f"Image is {size} bytes; the inline limit is "
+ f"{self._MAX_INLINE_MEDIA_BYTES} bytes."
+ ),
+ "success": False,
+ "media_kind": media_kind,
+ }
+ data = await client.download_bytes(uri)
+ if len(data) > size:
+ return {
+ "uri": uri,
+ "content": "Image changed after its size was checked; retry the read.",
+ "success": False,
+ "media_kind": media_kind,
+ }
+ image_format = sniff_image_format(data)
+ if (
+ image_format is None
+ or image_format.mime_type not in self._INLINE_IMAGE_MIME_TYPES
+ ):
+ return {
+ "uri": uri,
+ "content": (
+ "File bytes do not match a supported PNG, JPEG, GIF, "
+ "or WebP image."
+ ),
+ "success": False,
+ "media_kind": media_kind,
+ }
+ encoded = base64.b64encode(data).decode("ascii")
+ return {
+ "uri": uri,
+ "content": (
+ f"Image resource ({image_format.mime_type}, {len(data)} bytes)."
+ ),
+ "image": {
+ "type": "image_url",
+ "image_url": {
+ "url": f"data:{image_format.mime_type};base64,{encoded}"
+ },
+ },
+ "media_bytes": len(data),
+ "success": True,
+ "media_kind": media_kind,
+ }
+
+ if media_kind in {"audio", "video"}:
+ content = await client.read_content(uri, level="overview")
+ if not content:
+ return {
+ "uri": uri,
+ "content": (
+ "No OpenViking textual overview is available for this "
+ f"{media_kind} resource."
+ ),
+ "success": False,
+ "media_kind": media_kind,
+ }
+ return {
+ "uri": uri,
+ "content": content,
+ "success": True,
+ "media_kind": media_kind,
+ }
+
if limit == -1:
try:
stat = await client.stat(uri)
@@ -994,6 +1116,7 @@ class VikingMultiReadTool(OVFileTool):
"openviking_multi_read offset/limit."
),
"success": False,
+ "media_kind": None,
}
content = await client.read_content(
uri, level=level, offset=offset, limit=limit
@@ -1007,6 +1130,7 @@ class VikingMultiReadTool(OVFileTool):
"uri": uri,
"content": content,
"success": True,
+ "media_kind": None,
}
except Exception as e:
logger.warning(f"Error reading from {uri}: {e}")
@@ -1014,12 +1138,29 @@ class VikingMultiReadTool(OVFileTool):
"uri": uri,
"content": f"Error reading from Viking: {str(e)}",
"success": False,
+ "media_kind": self._media_kind(uri),
}
# 并发读取所有URI
- read_tasks = [read_single_uri(uri) for uri in uris]
+ read_tasks = [read_single_uri(index, uri) for index, uri in enumerate(uris)]
results = await asyncio.gather(*read_tasks)
+ inline_media_bytes = 0
+ for result in results:
+ if result.get("image") is None:
+ continue
+ media_bytes = result.get("media_bytes", 0)
+ if inline_media_bytes + media_bytes > self._MAX_INLINE_MEDIA_BYTES:
+ result.pop("image", None)
+ result["media_bytes"] = 0
+ result["content"] = (
+ "Image was not attached because the combined inline media size would "
+ f"exceed {self._MAX_INLINE_MEDIA_BYTES} bytes. Read fewer images at once."
+ )
+ result["success"] = False
+ continue
+ inline_media_bytes += media_bytes
+
# 构建结果
range_note = (
""
@@ -1027,20 +1168,45 @@ class VikingMultiReadTool(OVFileTool):
else f" (lines {offset}..{offset + limit - 1 if limit > 0 else 'end'})"
)
result_lines = [f"Multi-read results for {len(uris)} resources (level: {level}):"]
+ has_images = any(result.get("image") is not None for result in results)
+ content_parts: list[dict[str, Any]] = (
+ [{"type": "text", "text": result_lines[0]}] if has_images else []
+ )
for result in results:
uri = result["uri"]
content = result["content"]
success = result["success"]
+ item_range_note = "" if result["media_kind"] else range_note
- result_lines.append(f"\n--- START OF {uri}{range_note} ---")
+ start_marker = f"\n--- START OF {uri}{item_range_note} ---"
+ end_marker = f"--- END OF {uri} ---"
+ result_lines.append(start_marker)
if success:
result_lines.append(content)
else:
result_lines.append(f"ERROR: {content}")
- result_lines.append(f"--- END OF {uri} ---")
+ result_lines.append(end_marker)
- return "\n".join(result_lines)
+ if has_images:
+ content_parts.append(
+ {
+ "type": "text",
+ "text": f"{start_marker}\n{'ERROR: ' if not success else ''}{content}",
+ }
+ )
+ image = result.get("image")
+ if image is not None:
+ content_parts.append(image)
+ content_parts.append({"type": "text", "text": end_marker})
+
+ text_result = "\n".join(result_lines)
+ if has_images:
+ return MultimodalToolResult(
+ text=text_result,
+ content=content_parts,
+ )
+ return text_result
except Exception as e:
logger.exception(f"Error in VikingMultiReadTool: {e}")
@@ -1178,9 +1344,7 @@ class VikingExportTool(OVFileTool):
f"exceed the total-byte budget ({self._MAX_TOTAL_BYTES} bytes)."
)
return f"No text files exported from {uri}."
- lines = [
- f"Exported {len(exported)} file(s) into the workspace under '{dest}/':"
- ]
+ lines = [f"Exported {len(exported)} file(s) into the workspace under '{dest}/':"]
lines.extend(f"- {line}" for line in exported)
if skipped_binary:
lines.append(f"Skipped {skipped_binary} non-UTF-8 (binary) file(s).")
diff --git a/bot/vikingbot/agent/tools/registry.py b/bot/vikingbot/agent/tools/registry.py
index 395495bdf..debe0dbf0 100644
--- a/bot/vikingbot/agent/tools/registry.py
+++ b/bot/vikingbot/agent/tools/registry.py
@@ -8,7 +8,7 @@ from typing import Any
from loguru import logger
from vikingbot.agent.remote_skills import SkillRuntimeError
-from vikingbot.agent.tools.base import Tool, ToolContext
+from vikingbot.agent.tools.base import MultimodalToolResult, Tool, ToolContext
from vikingbot.config.schema import SessionKey
from vikingbot.hooks import HookContext
from vikingbot.hooks.manager import hook_manager
@@ -340,6 +340,8 @@ class ToolRegistry:
channel_metadata=channel_metadata,
skill_runtime=skill_runtime,
)
+ if isinstance(outcome.result, MultimodalToolResult):
+ return str(outcome.result)
return outcome.result
@property
diff --git a/bot/vikingbot/hooks/builtins/openviking_hooks.py b/bot/vikingbot/hooks/builtins/openviking_hooks.py
index ea40f4228..1c69502ca 100644
--- a/bot/vikingbot/hooks/builtins/openviking_hooks.py
+++ b/bot/vikingbot/hooks/builtins/openviking_hooks.py
@@ -409,12 +409,13 @@ class OpenVikingPostCallHook(Hook):
)
if is_skill_read:
if result and not isinstance(result, Exception):
- match = re.search(r"^---\s*\nname:\s*(.+?)\s*\n", result, re.MULTILINE)
+ result_text = str(result)
+ match = re.search(r"^---\s*\nname:\s*(.+?)\s*\n", result_text, re.MULTILINE)
if match:
skill_name = match.group(1).strip()
if skill_name == "experience_loader":
return {"tool_name": tool_name, "params": params, "result": result}
- desc_match = re.search(r"^description:\s*(.+)$", result, re.MULTILINE)
+ desc_match = re.search(r"^description:\s*(.+)$", result_text, re.MULTILINE)
skill_query = desc_match.group(1).strip() if desc_match else skill_name
exp_memory = await self._search_skill_experiences(
@@ -424,7 +425,7 @@ class OpenVikingPostCallHook(Hook):
openviking_connection=context.openviking_connection,
)
if exp_memory:
- result = f"{result}\n\n---\n## Related Experiences\n{exp_memory}"
+ result = f"{result_text}\n\n---\n## Related Experiences\n{exp_memory}"
return {"tool_name": tool_name, "params": params, "result": result}
diff --git a/bot/vikingbot/openviking_mount/ov_server.py b/bot/vikingbot/openviking_mount/ov_server.py
index 14a6fccee..6aa9a113c 100644
--- a/bot/vikingbot/openviking_mount/ov_server.py
+++ b/bot/vikingbot/openviking_mount/ov_server.py
@@ -597,7 +597,17 @@ class VikingClient:
return entries
async def stat(self, uri: str) -> Dict[str, Any]:
- return await self.client.stat(uri)
+ client = self.client
+ should_close = False
+ scoped_user_id = self._owner_user_id_for_uri(uri)
+ if scoped_user_id:
+ client, should_close = await self._get_user_scoped_client(scoped_user_id)
+
+ try:
+ return await client.stat(uri)
+ finally:
+ if should_close:
+ await client.close()
async def attrs(self, uri: str) -> Dict[str, Any]:
return await self.client.attrs(uri)
@@ -612,7 +622,17 @@ class VikingClient:
return await self.client.read_raw(uri, offset=offset, limit=limit)
async def download_bytes(self, uri: str) -> bytes:
- return await self.client.download_bytes(uri)
+ client = self.client
+ should_close = False
+ scoped_user_id = self._owner_user_id_for_uri(uri)
+ if scoped_user_id:
+ client, should_close = await self._get_user_scoped_client(scoped_user_id)
+
+ try:
+ return await client.download_bytes(uri)
+ finally:
+ if should_close:
+ await client.close()
async def find_skills(
self,
diff --git a/bot/vikingbot/providers/base.py b/bot/vikingbot/providers/base.py
index 7b1d16591..183d5660f 100644
--- a/bot/vikingbot/providers/base.py
+++ b/bot/vikingbot/providers/base.py
@@ -192,6 +192,10 @@ class LLMProvider(ABC):
)
yield LLMStreamEvent(type="response", response=response)
+ def supports_tool_result_media(self, model: str | None = None) -> bool:
+ """Whether this provider can preserve media inside a tool result."""
+ return False
+
@abstractmethod
def get_default_model(self) -> str:
"""Get the default model for this provider."""
diff --git a/bot/vikingbot/providers/litellm_provider.py b/bot/vikingbot/providers/litellm_provider.py
index 74d7d155b..70f07f411 100644
--- a/bot/vikingbot/providers/litellm_provider.py
+++ b/bot/vikingbot/providers/litellm_provider.py
@@ -563,3 +563,10 @@ class LiteLLMProvider(LLMProvider):
def get_default_model(self) -> str:
"""Get the default model."""
return self.default_model
+
+ def supports_tool_result_media(self, model: str | None = None) -> bool:
+ """Enable only the native tool-result media path verified for Anthropic."""
+ if self._gateway is not None:
+ return False
+ spec = find_by_model(model or self.default_model)
+ return spec is not None and spec.name == "anthropic"
diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.py
index 9787bf491..7600a2d57 100644
--- a/bot/vikingbot/providers/vlm_adapter.py
+++ b/bot/vikingbot/providers/vlm_adapter.py
@@ -721,6 +721,32 @@ class VLMProviderAdapter(LLMProvider):
def get_default_model(self) -> str:
return self._default_model
+ def supports_tool_result_media(self, model: str | None = None) -> bool:
+ """Return true only for backends with native rich tool-result support."""
+ return self._backend_supports_tool_result_media(self._vlm)
+
+ @classmethod
+ def _backend_supports_tool_result_media(cls, vlm: Any) -> bool:
+ instances = getattr(vlm, "_vlm_instances", None)
+ if instances is not None:
+ return bool(instances) and all(
+ cls._backend_supports_tool_result_media(item) for item in instances
+ )
+
+ primary = getattr(vlm, "primary", None)
+ backup = getattr(vlm, "backup", None)
+ if primary is not None and backup is not None:
+ return cls._backend_supports_tool_result_media(
+ primary
+ ) and cls._backend_supports_tool_result_media(backup)
+
+ resolve_provider = getattr(vlm, "resolved_provider", None)
+ provider = (
+ resolve_provider() if callable(resolve_provider) else getattr(vlm, "provider", "")
+ )
+ provider = str(provider or "").lower()
+ return provider in {"anthropic", "openai-codex"}
+
# ------------------------------------------------------------------
# Langfuse helpers (same pattern as LiteLLMProvider.chat())
# ------------------------------------------------------------------
diff --git a/bot/vikingbot/tests/unit/test_bot_provider_thinking.py b/bot/vikingbot/tests/unit/test_bot_provider_thinking.py
index ae61f4315..d05a4c36f 100644
--- a/bot/vikingbot/tests/unit/test_bot_provider_thinking.py
+++ b/bot/vikingbot/tests/unit/test_bot_provider_thinking.py
@@ -4,6 +4,9 @@ from types import SimpleNamespace
import pytest
+from openviking.models.vlm.backends.litellm_vlm import (
+ LiteLLMVLMProvider as OpenVikingLiteLLMVLMProvider,
+)
from vikingbot.config.schema import AgentsConfig
from vikingbot.providers.litellm_provider import LiteLLMProvider
from vikingbot.providers.vlm_adapter import VLMProviderAdapter
@@ -29,6 +32,73 @@ def test_agents_openviking_retention_defaults_to_turn_budget_values():
assert config.commit_min_raw_tail_steps == 1
+def test_vlm_adapter_exposes_only_native_tool_result_media_backends():
+ langfuse = SimpleNamespace()
+
+ codex = VLMProviderAdapter(
+ SimpleNamespace(provider="openai-codex"),
+ default_model="gpt-5.3-codex",
+ langfuse_client=langfuse,
+ )
+ anthropic = VLMProviderAdapter(
+ SimpleNamespace(provider="anthropic"),
+ default_model="claude-sonnet",
+ langfuse_client=langfuse,
+ )
+ openai = VLMProviderAdapter(
+ SimpleNamespace(provider="openai"),
+ default_model="gpt-4o",
+ langfuse_client=langfuse,
+ )
+
+ assert codex.supports_tool_result_media() is True
+ assert anthropic.supports_tool_result_media() is True
+ assert openai.supports_tool_result_media() is False
+
+ mixed_failover = VLMProviderAdapter(
+ SimpleNamespace(
+ provider="anthropic",
+ primary=SimpleNamespace(provider="anthropic"),
+ backup=SimpleNamespace(provider="openai"),
+ ),
+ default_model="claude-sonnet",
+ langfuse_client=langfuse,
+ )
+ assert mixed_failover.supports_tool_result_media() is False
+
+
+def test_litellm_provider_exposes_anthropic_tool_result_media_only():
+ langfuse = SimpleNamespace()
+ anthropic = LiteLLMProvider(
+ default_model="claude-sonnet-4-5",
+ langfuse_client=langfuse,
+ )
+ openai = LiteLLMProvider(
+ default_model="gpt-4o",
+ langfuse_client=langfuse,
+ )
+
+ assert anthropic.supports_tool_result_media() is True
+ assert openai.supports_tool_result_media() is False
+
+
+def test_vlm_adapter_uses_litellm_resolved_provider_for_tool_result_media():
+ vlm = OpenVikingLiteLLMVLMProvider(
+ {
+ "provider": "litellm",
+ "model": "claude-sonnet-4-5",
+ }
+ )
+ provider = VLMProviderAdapter(
+ vlm,
+ default_model="claude-sonnet-4-5",
+ langfuse_client=SimpleNamespace(),
+ )
+
+ assert vlm.resolved_provider() == "anthropic"
+ assert provider.supports_tool_result_media() is True
+
+
def test_make_provider_passes_default_thinking_to_vlm_adapter(monkeypatch):
monkeypatch.setitem(
sys.modules,
diff --git a/bot/vikingbot/utils/image_format.py b/bot/vikingbot/utils/image_format.py
index 8ec7bfff5..08ea6978e 100644
--- a/bot/vikingbot/utils/image_format.py
+++ b/bot/vikingbot/utils/image_format.py
@@ -27,8 +27,8 @@ def image_format_from_mime(mime_type: str | None) -> ImageFormat | None:
return _MIME_FORMATS.get(mime_type.lower().split(";", 1)[0].strip())
-def detect_image_format(image_data: bytes, fallback_mime: str | None = None) -> ImageFormat:
- """Detect common image formats from magic bytes, falling back to a MIME hint."""
+def sniff_image_format(image_data: bytes) -> ImageFormat | None:
+ """Detect common image formats from magic bytes."""
if image_data.startswith(b"\x89PNG\r\n\x1a\n"):
return PNG_FORMAT
if image_data.startswith(b"\xff\xd8\xff"):
@@ -40,4 +40,13 @@ def detect_image_format(image_data: bytes, fallback_mime: str | None = None) ->
if len(image_data) >= 12 and image_data[:4] == b"RIFF" and image_data[8:12] == b"WEBP":
return ImageFormat("webp", "image/webp")
+ return None
+
+
+def detect_image_format(image_data: bytes, fallback_mime: str | None = None) -> ImageFormat:
+ """Detect common image formats from magic bytes, falling back to a MIME hint."""
+ detected = sniff_image_format(image_data)
+ if detected is not None:
+ return detected
+
return image_format_from_mime(fallback_mime) or PNG_FORMAT
diff --git a/openviking/models/vlm/backends/codex_responses_adapter.py b/openviking/models/vlm/backends/codex_responses_adapter.py
index d1bf0e3a7..f650de40c 100644
--- a/openviking/models/vlm/backends/codex_responses_adapter.py
+++ b/openviking/models/vlm/backends/codex_responses_adapter.py
@@ -86,6 +86,16 @@ def _stringify_response_payload(value: Any, *, default: str = "") -> str:
return str(value)
+def _convert_tool_output_for_responses(content: Any) -> Any:
+ if isinstance(content, list):
+ converted = _convert_content_for_responses(content)
+ if isinstance(converted, list) and any(
+ part.get("type") == "input_image" for part in converted if isinstance(part, dict)
+ ):
+ return converted
+ return _stringify_response_payload(content)
+
+
def _convert_tool_call_history(tool_calls: Any) -> List[Dict[str, Any]]:
if not isinstance(tool_calls, list):
return []
@@ -118,7 +128,7 @@ def _convert_message_for_responses(message: Dict[str, Any]) -> List[Dict[str, An
{
"type": "function_call_output",
"call_id": str(message.get("tool_call_id", "") or message.get("call_id", "") or ""),
- "output": _stringify_response_payload(content),
+ "output": _convert_tool_output_for_responses(content),
}
]
if role == "assistant":
diff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.py
index cbd7c56e0..84e9057a8 100644
--- a/openviking/models/vlm/backends/litellm_vlm.py
+++ b/openviking/models/vlm/backends/litellm_vlm.py
@@ -211,6 +211,13 @@ class LiteLLMVLMProvider(VLMBase):
return model
+ def resolved_provider(self, model: str | None = None) -> str | None:
+ """Return the provider LiteLLM will route this model to."""
+ resolved_model = self._resolve_model(model or self.model or "")
+ if "/" in resolved_model:
+ return resolved_model.split("/", 1)[0].lower()
+ return self._detected_provider or detect_provider_by_model(resolved_model) or self.provider
+
def _should_forward_api_key(self, model: str) -> bool:
if not self.api_key:
return False
diff --git a/openviking/server/mcp_endpoint.py b/openviking/server/mcp_endpoint.py
index 45646dc55..c732a8b3f 100644
--- a/openviking/server/mcp_endpoint.py
+++ b/openviking/server/mcp_endpoint.py
@@ -65,6 +65,7 @@ from openviking.server.local_input_guard import (
from openviking.server.resource_ingest import ingest_temp_upload
from openviking.server.temp_upload_store import TempUploadStore
from openviking.server.upload_token_store import upload_token_store
+from openviking.utils.media_limits import MAX_INLINE_TOOL_RESULT_MEDIA_BYTES
from openviking.utils.search_filters import SearchContextTypeInput, merge_search_filter
from openviking_cli.exceptions import (
InvalidArgumentError,
@@ -425,9 +426,7 @@ async def _format_search_result(result, *, service, ctx, read_content: bool = Fa
_MCP_IMAGE_EXTENSIONS = {".gif", ".jpeg", ".jpg", ".png", ".webp"}
_MCP_AUDIO_EXTENSIONS = {".flac", ".m4a", ".mp3", ".oga", ".ogg", ".wav"}
_MCP_VIDEO_EXTENSIONS = {".avi", ".m4v", ".mkv", ".mov", ".mp4", ".webm"}
-# Common clients cap an inline result at 5 MiB base64, or 3.75 MiB raw.
-# Apply the same limit to one file and to the aggregate media in one tool call.
-_MCP_MEDIA_MAX_BYTES = 3_932_160
+_MCP_MEDIA_MAX_BYTES = MAX_INLINE_TOOL_RESULT_MEDIA_BYTES
def _mcp_uri_suffix(uri: str) -> str:
diff --git a/openviking/utils/media_limits.py b/openviking/utils/media_limits.py
index c64fbba3d..dff555fc0 100644
--- a/openviking/utils/media_limits.py
+++ b/openviking/utils/media_limits.py
@@ -5,6 +5,9 @@
from typing import Protocol
MAX_MEDIA_FILE_BYTES = 512 * 1024 * 1024
+# Common model clients cap one inline tool result at 5 MiB after Base64 encoding.
+# Apply the equivalent raw-byte budget to both individual media and aggregates.
+MAX_INLINE_TOOL_RESULT_MEDIA_BYTES = 3_932_160
DEFAULT_LARGE_IMAGE_MAX_FILE_SIZE_MB = 10.0
DEFAULT_LARGE_IMAGE_THRESHOLD_DIMENSION = 4096
diff --git a/tests/unit/test_codex_vlm.py b/tests/unit/test_codex_vlm.py
index 10d40987b..664b7012f 100644
--- a/tests/unit/test_codex_vlm.py
+++ b/tests/unit/test_codex_vlm.py
@@ -10,6 +10,9 @@ import pytest
from openviking.models.vlm.backends import codex_auth
from openviking.models.vlm.backends.codex_auth import resolve_codex_runtime_credentials
+from openviking.models.vlm.backends.codex_responses_adapter import (
+ _convert_message_for_responses,
+)
from openviking.models.vlm.backends.codex_vlm import CodexVLM
from openviking_cli.utils.config.vlm_config import VLMConfig
@@ -568,6 +571,89 @@ def test_codex_translates_tool_history_into_responses_input(mock_resolve, mock_o
]
+ ("content", "expected_output"),
+ [
+ (
+ [
+ {"type": "text", "text": "first"},
+ {"type": "text", "text": "second"},
+ ],
+ "firstsecond",
+ ),
+ ({"ok": True}, '{"ok": true}'),
+ ],
+)
+def test_codex_preserves_legacy_text_tool_output_serialization(content, expected_output):
+ converted = _convert_message_for_responses(
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": content,
+ }
+ )
+
+ assert converted == [
+ {
+ "type": "function_call_output",
+ "call_id": "call_1",
+ "output": expected_output,
+ }
+ ]
+
+
+@patch("openviking.models.vlm.backends.codex_vlm.openai.OpenAI")
+@patch("openviking.models.vlm.backends.codex_vlm.resolve_codex_runtime_credentials")
+def test_codex_preserves_images_in_function_call_output(mock_resolve, mock_openai_class):
+ mock_resolve.return_value = {
+ "api_key": "oauth-token",
+ "base_url": "https://chatgpt.com/backend-api/codex",
+ }
+ mock_real_client = MagicMock()
+ mock_real_client.responses.create.return_value = _MockResponsesStream(
+ _build_final_response("image inspected")
+ )
+ mock_openai_class.return_value = mock_real_client
+ image_url = "data:image/png;base64,aW1hZ2U="
+
+ vlm = CodexVLM({"provider": "openai-codex", "model": "gpt-5.3-codex"})
+ result = vlm.get_completion(
+ messages=[
+ {"role": "user", "content": "Inspect the image"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "read_image", "arguments": "{}"},
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "tool_call_id": "call_1",
+ "content": [
+ {"type": "text", "text": "Source: image.png"},
+ {"type": "image_url", "image_url": {"url": image_url}},
+ ],
+ },
+ ]
+ )
+
+ assert result == "image inspected"
+ input_items = mock_real_client.responses.create.call_args.kwargs["input"]
+ assert input_items[-1] == {
+ "type": "function_call_output",
+ "call_id": "call_1",
+ "output": [
+ {"type": "input_text", "text": "Source: image.png"},
+ {"type": "input_image", "image_url": image_url},
+ ],
+ }
+
+
def test_codex_auth_invalid_exp_claim_is_treated_as_expiring():
assert codex_auth._codex_access_token_is_expiring("not-a-jwt", skew_seconds=60) is True
assert (