Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(bot): support multimodal OpenViking resource reads (#4593)
DuTao6020c62ccc2c
diff --git a/bot/tests/test_agent_loop_outcome.py b/bot/tests/test_agent_loop_outcome.pyindex da49eed79..7b8f17806 100644--- a/bot/tests/test_agent_loop_outcome.py+++ b/bot/tests/test_agent_loop_outcome.py@@ -1,3 +1,4 @@+import copyimport sysfrom datetime import datetimefrom pathlib import Path@@ -9,6 +10,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1]))from vikingbot.agent import loop as loop_modulefrom vikingbot.agent.context import ContextBuilderfrom vikingbot.agent.loop import AgentLoop+from vikingbot.agent.tools.base import MultimodalToolResult+from vikingbot.agent.tools.registry import ToolExecutionResultfrom vikingbot.bus.events import InboundMessage, OutboundEventTypefrom vikingbot.bus.queue import MessageBusfrom vikingbot.config.schema import AgentsConfig, Config, SessionKey@@ -108,6 +111,305 @@ def test_agents_config_keeps_ten_recent_openviking_messages_by_default():assert AgentsConfig().commit_keep_recent_count == 10+def test_context_keeps_multimodal_tool_result_on_tool_message(temp_dir: Path):+ context = ContextBuilder(workspace=temp_dir / "workspace")+ content = [+ {"type": "text", "text": "Source: viking://resources/image.png"},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,aW1hZ2U="},+ },+ ]+ messages = context.add_tool_result(+ [],+ "call-1",+ "openviking_multi_read",+ MultimodalToolResult(text="Image resource.", content=content),+ )++ assert messages == [+ {+ "role": "tool",+ "tool_call_id": "call-1",+ "name": "openviking_multi_read",+ "content": content,+ }+ ]++[email protected]("supports_media", [False, True])+async def test_agent_loop_gates_multimodal_result_by_provider(+ temp_dir: Path, monkeypatch, supports_media: bool+):+ monkeypatch.setattr(AgentLoop, "_register_builtin_hooks", lambda self: None)+ monkeypatch.setattr(AgentLoop, "_register_default_tools", lambda self: None)+ monkeypatch.setattr("vikingbot.agent.loop.SubagentManager", _FakeSubagentManager)++ content = [+ {"type": "text", "text": "Source: viking://resources/image.png"},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,aW1hZ2U="},+ },+ ]++ class Provider(LLMProvider):+ def __init__(self):+ super().__init__()+ self.calls = []++ async def chat(self, messages, tools=None, **kwargs):+ self.calls.append(messages)+ if len(self.calls) == 1:+ return LLMResponse(+ content=None,+ tool_calls=[+ ToolCallRequest(+ id="call-1",+ name="read_image",+ arguments={},+ tokens=1,+ )+ ],+ )+ return LLMResponse(content="done")++ def get_default_model(self) -> str:+ return "fake-model"++ def supports_tool_result_media(self, model=None) -> bool:+ return supports_media++ class Registry:+ def get_definitions(self, **kwargs):+ return [+ {+ "type": "function",+ "function": {+ "name": "read_image",+ "description": "Read an image",+ "parameters": {"type": "object", "properties": {}},+ },+ }+ ]++ async def execute_detailed(self, name, params, **kwargs):+ return ToolExecutionResult(+ result=MultimodalToolResult(text="Image resource.", content=content),+ effective_params=params,+ )++ provider = Provider()+ loop = AgentLoop(+ bus=MessageBus(),+ provider=provider,+ workspace=temp_dir / "workspace",+ config=Config(storage_workspace=str(temp_dir)),+ max_iterations=2,+ )++ final, _reasoning, tools_used, _usage, _iteration = await loop._run_agent_loop(+ messages=[{"role": "user", "content": "read it"}],+ session_key=SessionKey(type="cli", channel_id="default", chat_id="multimodal"),+ publish_events=False,+ tool_registry=Registry(),+ )++ assert final == "done"+ tool_message = next(message for message in provider.calls[1] if message["role"] == "tool")+ if supports_media:+ assert tool_message["content"] == content+ else:+ assert tool_message["content"].startswith("Image resource.")+ assert "does not support media in tool results" in tool_message["content"]+ assert tools_used[0]["result"] == "Image resource."+++async def test_agent_loop_limits_media_across_parallel_tool_results(temp_dir: Path, monkeypatch):+ monkeypatch.setattr(AgentLoop, "_register_builtin_hooks", lambda self: None)+ monkeypatch.setattr(AgentLoop, "_register_default_tools", lambda self: None)+ monkeypatch.setattr("vikingbot.agent.loop.SubagentManager", _FakeSubagentManager)+ monkeypatch.setattr(loop_module, "MAX_INLINE_TOOL_RESULT_MEDIA_BYTES", 5)++ class Provider(LLMProvider):+ def __init__(self):+ super().__init__()+ self.calls = []++ async def chat(self, messages, tools=None, **kwargs):+ self.calls.append(messages)+ if len(self.calls) == 1:+ return LLMResponse(+ content=None,+ tool_calls=[+ ToolCallRequest(+ id="call-1",+ name="read_image",+ arguments={"label": "first"},+ tokens=1,+ ),+ ToolCallRequest(+ id="call-2",+ name="read_image",+ arguments={"label": "second"},+ tokens=1,+ ),+ ],+ )+ return LLMResponse(content="done")++ def get_default_model(self) -> str:+ return "fake-model"++ def supports_tool_result_media(self, model=None) -> bool:+ return True++ class Registry:+ def get_definitions(self, **kwargs):+ return [+ {+ "type": "function",+ "function": {+ "name": "read_image",+ "description": "Read an image",+ "parameters": {"type": "object", "properties": {}},+ },+ }+ ]++ async def execute_detailed(self, name, params, **kwargs):+ label = params["label"]+ return ToolExecutionResult(+ result=MultimodalToolResult(+ text=f"{label} image",+ content=[+ {"type": "text", "text": label},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,ZGF0YQ=="},+ },+ ],+ ),+ effective_params=params,+ )++ provider = Provider()+ loop = AgentLoop(+ bus=MessageBus(),+ provider=provider,+ workspace=temp_dir / "workspace",+ config=Config(storage_workspace=str(temp_dir)),+ max_iterations=2,+ )++ final, *_ = await loop._run_agent_loop(+ messages=[{"role": "user", "content": "read both"}],+ session_key=SessionKey(type="cli", channel_id="default", chat_id="media-budget"),+ publish_events=False,+ tool_registry=Registry(),+ )++ assert final == "done"+ tool_messages = [message for message in provider.calls[1] if message["role"] == "tool"]+ assert isinstance(tool_messages[0]["content"], list)+ assert isinstance(tool_messages[1]["content"], str)+ assert "make this model request exceed" in tool_messages[1]["content"]+++async def test_agent_loop_prefers_new_media_across_consecutive_tool_rounds(+ temp_dir: Path, monkeypatch+):+ monkeypatch.setattr(AgentLoop, "_register_builtin_hooks", lambda self: None)+ monkeypatch.setattr(AgentLoop, "_register_default_tools", lambda self: None)+ monkeypatch.setattr("vikingbot.agent.loop.SubagentManager", _FakeSubagentManager)+ monkeypatch.setattr(loop_module, "MAX_INLINE_TOOL_RESULT_MEDIA_BYTES", 5)++ class Provider(LLMProvider):+ def __init__(self):+ super().__init__()+ self.calls = []++ async def chat(self, messages, tools=None, **kwargs):+ self.calls.append(copy.deepcopy(messages))+ call_number = len(self.calls)+ if call_number <= 2:+ return LLMResponse(+ content=None,+ tool_calls=[+ ToolCallRequest(+ id=f"call-{call_number}",+ name="read_image",+ arguments={"label": f"image-{call_number}"},+ tokens=1,+ )+ ],+ )+ return LLMResponse(content="done")++ def get_default_model(self) -> str:+ return "fake-model"++ def supports_tool_result_media(self, model=None) -> bool:+ return True++ class Registry:+ def get_definitions(self, **kwargs):+ return [+ {+ "type": "function",+ "function": {+ "name": "read_image",+ "description": "Read an image",+ "parameters": {"type": "object", "properties": {}},+ },+ }+ ]++ async def execute_detailed(self, name, params, **kwargs):+ label = params["label"]+ return ToolExecutionResult(+ result=MultimodalToolResult(+ text=f"{label} result",+ content=[+ {"type": "text", "text": label},+ {+ "type": "image_url",+ "image_url": {"url": "data:image/png;base64,ZGF0YQ=="},+ },+ ],+ ),+ effective_params=params,+ )++ provider = Provider()+ loop = AgentLoop(+ bus=MessageBus(),+ provider=provider,+ workspace=temp_dir / "workspace",+ config=Config(storage_workspace=str(temp_dir)),+ max_iterations=3,+ )++ final, *_ = await loop._run_agent_loop(+ messages=[{"role": "user", "content": "read two images in sequence"}],+ session_key=SessionKey(type="cli", channel_id="default", chat_id="media-rounds"),+ publish_events=False,+ tool_registry=Registry(),+ )++ assert final == "done"+ first_round_tool = next(message for message in provider.calls[1] if message["role"] == "tool")+ assert isinstance(first_round_tool["content"], list)++ tool_messages = [message for message in provider.calls[-1] if message["role"] == "tool"]+ assert isinstance(tool_messages[0]["content"], str)+ assert "earlier tool result was omitted" in tool_messages[0]["content"]+ assert isinstance(tool_messages[1]["content"], list)++def test_agent_loop_omits_spawn_tool_when_subagents_disabled(temp_dir: Path, monkeypatch):monkeypatch.setattr(AgentLoop, "_register_builtin_hooks", lambda self: None)monkeypatch.setattr("vikingbot.agent.loop.SubagentManager", _FakeSubagentManager)diff --git a/bot/tests/test_compile.py b/bot/tests/test_compile.pyindex 5e424e504..facc5bfa6 100644--- a/bot/tests/test_compile.py+++ b/bot/tests/test_compile.py@@ -11,7 +11,7 @@ from vikingbot.agent.loop import (AgentLoop,render_budget_reminder,)-from vikingbot.agent.tools.base import Tool, ToolContext+from vikingbot.agent.tools.base import MultimodalToolResult, Tool, ToolContextfrom vikingbot.agent.tools.compile import (CompileScopedTool,SubmitTargetCheckoutTool,@@ -1366,6 +1366,146 @@ async def test_multi_read_hints_on_large_files_and_supports_offset_limit(monkeypassert fake.calls == [("viking://resources/big.jsonl", 10, 20)]+async def test_multi_read_returns_images_as_tool_result_content(monkeypatch):+ image_uri = "viking://resources/example/image.png"+ image_bytes = b"\x89PNG\r\n\x1a\nimage-data"++ class FakeClient:+ async def stat(self, uri):+ assert uri == image_uri+ return {"size": len(image_bytes), "isDir": False}++ async def download_bytes(self, uri):+ assert uri == image_uri+ return image_bytes++ async def read_content(self, *args, **kwargs):+ raise AssertionError("image reads must not use the text endpoint")++ tool = VikingMultiReadTool()++ async def fake_get_client(ctx):+ return FakeClient()++ async def no_release(ctx, client):+ return None++ monkeypatch.setattr(tool, "_get_client", fake_get_client)+ monkeypatch.setattr(tool, "_release_client", no_release)++ result = await tool.execute(ToolContext(), uris=[image_uri])++ assert isinstance(result, MultimodalToolResult)+ assert f"START OF {image_uri}" in str(result)+ assert "Image resource (image/png" in str(result)+ assert result.content[2] == {+ "type": "image_url",+ "image_url": {+ "url": f"data:image/png;base64,{base64.b64encode(image_bytes).decode('ascii')}"+ },+ }+++async def test_multi_read_treats_typescript_as_text_and_preserves_window(monkeypatch):+ typescript_uri = "viking://resources/web/index.ts"++ class FakeClient:+ async def read_content(self, uri, level="abstract", offset=0, limit=-1, **kwargs):+ assert uri == typescript_uri+ assert level == "read"+ assert (offset, limit) == (4, 2)+ return "export const answer = 42;"++ async def download_bytes(self, uri):+ raise AssertionError("TypeScript reads must not use the media download path")++ tool = VikingMultiReadTool()++ async def fake_get_client(ctx):+ return FakeClient()++ async def no_release(ctx, client):+ return None++ monkeypatch.setattr(tool, "_get_client", fake_get_client)+ monkeypatch.setattr(tool, "_release_client", no_release)++ result = await tool.execute(ToolContext(), uris=[typescript_uri], offset=4, limit=2)++ assert isinstance(result, str)+ assert "export const answer = 42;" in result+++async def test_multi_read_limits_aggregate_inline_image_bytes(monkeypatch):+ first_uri = "viking://resources/first.png"+ second_uri = "viking://resources/second.png"+ first_image = b"\x89PNG\r\n\x1a\nfirst"+ second_image = b"\x89PNG\r\n\x1a\nother"+ images = {first_uri: first_image, second_uri: second_image}++ class FakeClient:+ async def stat(self, uri):+ return {"size": len(images[uri]), "isDir": False}++ async def download_bytes(self, uri):+ return images[uri]++ async def read_content(self, *args, **kwargs):+ raise AssertionError("image reads must not use the text endpoint")++ tool = VikingMultiReadTool()+ monkeypatch.setattr(tool, "_MAX_INLINE_MEDIA_BYTES", len(first_image))++ async def fake_get_client(ctx):+ return FakeClient()++ async def no_release(ctx, client):+ return None++ monkeypatch.setattr(tool, "_get_client", fake_get_client)+ monkeypatch.setattr(tool, "_release_client", no_release)++ result = await tool.execute(ToolContext(), uris=[first_uri, second_uri])++ assert isinstance(result, MultimodalToolResult)+ assert sum(part.get("type") == "image_url" for part in result.content) == 1+ assert "combined inline media size" in str(result)++[email protected]("extension", ["mp3", "mp4"])+async def test_multi_read_uses_openviking_overview_for_audio_and_video(monkeypatch, extension):+ media_uri = f"viking://resources/interview/interview.{extension}"++ class FakeClient:+ async def read_content(self, uri, level="abstract", **kwargs):+ assert uri == media_uri+ assert level == "overview"+ return "Speaker: hello"++ async def download_bytes(self, uri):+ raise AssertionError("media reads must not create a second transcription path")++ tool = VikingMultiReadTool()++ async def fake_get_client(ctx):+ return FakeClient()++ async def no_release(ctx, client):+ return None++ monkeypatch.setattr(tool, "_get_client", fake_get_client)+ monkeypatch.setattr(tool, "_release_client", no_release)++ result = await tool.execute(ToolContext(), uris=[media_uri])++ assert isinstance(result, str)+ assert "Speaker: hello" in result++class _RecordingSandbox:def __init__(self):self.writes = []diff --git a/bot/tests/test_image_format.py b/bot/tests/test_image_format.pyindex 4dfc27efb..71a3e8337 100644--- a/bot/tests/test_image_format.py+++ b/bot/tests/test_image_format.py@@ -7,7 +7,7 @@ from pathlib import Pathsys.path.insert(0, str(Path(__file__).resolve().parents[1]))-from vikingbot.utils.image_format import detect_image_format # noqa: E402+from vikingbot.utils.image_format import detect_image_format, sniff_image_format # noqa: E402def test_detect_image_format_identifies_jpeg():@@ -22,3 +22,7 @@ def test_detect_image_format_uses_mime_hint_for_unknown_bytes():assert image_format.extension == "webp"assert image_format.mime_type == "image/webp"+++def test_sniff_image_format_rejects_unknown_bytes():+ assert sniff_image_format(b"not an image") is Nonediff --git a/bot/tests/test_openviking_api_key_type.py b/bot/tests/test_openviking_api_key_type.pyindex 0ef698403..3d51a3c26 100644--- a/bot/tests/test_openviking_api_key_type.py+++ b/bot/tests/test_openviking_api_key_type.py@@ -48,6 +48,8 @@ class _DummyHTTPClient:self.find_calls = []self.ls_calls = []self.read_calls = []+ self.stat_calls = []+ self.download_calls = []self.closed = False_DummyHTTPClient.instances.append(self)@@ -105,6 +107,14 @@ class _DummyHTTPClient:self.read_calls.append(("read", uri))return ""+ async def stat(self, uri):+ self.stat_calls.append(uri)+ return {"size": 8, "isDir": False}++ async def download_bytes(self, uri):+ self.download_calls.append(uri)+ return b"image"+async def grep(self, *_args, **_kwargs):return {"matches": []}@@ -1958,6 +1968,27 @@ async def test_read_content_trusted_owner_uri_uses_owner_identity(monkeypatch):assert scoped.closed is True+async def test_image_reads_trusted_owner_uri_use_owner_identity(monkeypatch):+ monkeypatch.setattr(ov_server_module, "load_config", lambda: _make_config("root"))+ client = VikingClient()+ uri = "viking://user/sender-1/resources/image.png"++ stat = await client.stat(uri)+ content = await client.download_bytes(uri)++ assert stat == {"size": 8, "isDir": False}+ assert content == b"image"+ stat_client, download_client = _DummyHTTPClient.instances[1:]+ for scoped in (stat_client, download_client):+ assert scoped.kwargs["api_key"] == "root-key"+ assert scoped.kwargs["account"] == "acct"+ assert scoped.kwargs["user"] == "sender-1"+ assert scoped.closed is True+ assert stat_client.stat_calls == [uri]+ assert download_client.download_calls == [uri]++@pytest.mark.asyncioasync def test_search_memory_peer_ids_use_explicit_peer_uris(monkeypatch):monkeypatch.setattr(ov_server_module, "load_config", lambda: _make_config("user"))diff --git a/bot/vikingbot/agent/context.py b/bot/vikingbot/agent/context.pyindex dc188df0f..f317b6153 100644--- a/bot/vikingbot/agent/context.py+++ b/bot/vikingbot/agent/context.py@@ -12,6 +12,7 @@ from loguru import loggerfrom vikingbot.agent.memory import MemoryStorefrom vikingbot.agent.skills import SkillsLoader+from vikingbot.agent.tools.base import MultimodalToolResultfrom vikingbot.config.schema import SessionKeyfrom vikingbot.sandbox import SandboxManagerfrom vikingbot.utils.helpers import ensure_non_empty_assistant_content@@ -510,7 +511,7 @@ IMPORTANT:return images + [{"type": "text", "text": text}]def add_tool_result(- self, messages: list[dict[str, Any]], tool_call_id: str, tool_name: str, result: str+ self, messages: list[dict[str, Any]], tool_call_id: str, tool_name: str, result: Any) -> list[dict[str, Any]]:"""Add a tool result to the message list.@@ -524,8 +525,9 @@ IMPORTANT:Returns:Updated message list."""+ content = result.content if isinstance(result, MultimodalToolResult) else resultmessages.append(- {"role": "tool", "tool_call_id": tool_call_id, "name": tool_name, "content": result}+ {"role": "tool", "tool_call_id": tool_call_id, "name": tool_name, "content": content})return messagesdiff --git a/bot/vikingbot/agent/loop.py b/bot/vikingbot/agent/loop.pyindex 91bc6a73b..1c860c70a 100644--- a/bot/vikingbot/agent/loop.py+++ b/bot/vikingbot/agent/loop.py@@ -15,12 +15,14 @@ from typing import TYPE_CHECKING, Anyfrom loguru import logger+from openviking.utils.media_limits import MAX_INLINE_TOOL_RESULT_MEDIA_BYTESfrom vikingbot.agent.context import ContextBuilderfrom vikingbot.agent.memory import MemoryStorefrom vikingbot.agent.remote_skills import SkillRuntimeContextfrom vikingbot.agent.skills import SkillsLoaderfrom vikingbot.agent.subagent import SubagentManagerfrom vikingbot.agent.tools import register_default_tools+from vikingbot.agent.tools.base import MultimodalToolResultfrom vikingbot.agent.tools.registry import ToolExecutionResult, ToolRegistryfrom vikingbot.bus.events import InboundMessage, OutboundEventType, OutboundMessagefrom vikingbot.bus.queue import MessageBus@@ -59,6 +61,75 @@ def _compact_msg_chars(messages: list[dict]) -> int:return sum(len(json.dumps(m, ensure_ascii=False, default=str)) for m in messages)+def _base64_data_url_bytes(url: str) -> int:+ if not url.startswith("data:"):+ return 0+ marker = ";base64,"+ marker_index = url.find(marker)+ if marker_index < 0:+ return 0+ encoded_length = len(url) - marker_index - len(marker)+ if encoded_length <= 0:+ return 0+ padding = 2 if url.endswith("==") else 1 if url.endswith("=") else 0+ return max(0, encoded_length * 3 // 4 - padding)+++def _inline_media_bytes(messages: list[dict]) -> int:+ total = 0+ for message in messages:+ content = message.get("content")+ if not isinstance(content, list):+ continue+ for part in content:+ if not isinstance(part, dict) or part.get("type") not in {+ "image_url",+ "input_image",+ }:+ continue+ image_url = part.get("image_url")+ url = image_url.get("url") if isinstance(image_url, dict) else image_url+ if isinstance(url, str):+ total += _base64_data_url_bytes(url)+ return total+++def _demote_historical_tool_media(+ messages: list[dict], *, history_end: int, bytes_needed: int+) -> int:+ candidates: list[tuple[dict, int]] = []+ reclaimable_bytes = 0+ for message in messages[:history_end]:+ if message.get("role") != "tool" or not isinstance(message.get("content"), list):+ continue+ media_bytes = _inline_media_bytes([message])+ if media_bytes <= 0:+ continue+ candidates.append((message, media_bytes))+ reclaimable_bytes += media_bytes+ if reclaimable_bytes >= bytes_needed:+ break++ if reclaimable_bytes < bytes_needed:+ return 0++ for message, _media_bytes in candidates:+ text_parts = [+ part["text"]+ for part in message["content"]+ if isinstance(part, dict)+ and part.get("type") in {"text", "input_text"}+ and isinstance(part.get("text"), str)+ ]+ text = "\n".join(text_parts)+ omission_note = (+ "Media content from this earlier tool result was omitted from subsequent "+ "requests to keep the inline media budget."+ )+ message["content"] = f"{text}\n\n{omission_note}" if text else omission_note+ return reclaimable_bytes++def _compact_render_message(message: dict) -> str:role = message.get("role")content = message.get("content")@@ -136,9 +207,9 @@ def _compact_split_chunks(text: str) -> list[str]:def _compact_strip_note_header(content: str) -> str:text = str(content or "").strip()if text.startswith("[Context compaction]"):- text = text[len("[Context compaction]"):].strip()+ text = text[len("[Context compaction]") :].strip()if text.startswith("Findings so far:"):- text = text[len("Findings so far:"):].strip()+ text = text[len("Findings so far:") :].strip()return text.strip()@@ -1594,21 +1665,59 @@ class AgentLoop:# Stage 3: Process results sequentially in original orderturn_tools: list[dict[str, Any]] = []+ history_end = len(messages)+ request_media_bytes = _inline_media_bytes(messages)for _idx, tool_call, outcome, tool_execute_duration in results:result = outcome.result+ result_text = str(result)+ recorded_result = (+ result_text if isinstance(result, MultimodalToolResult) else result+ )args_str = json.dumps(tool_call.arguments, ensure_ascii=False)- logger.info(f"[RESULT]: {str(result)[:600]}")+ logger.info(f"[RESULT]: {result_text[:600]}")if publish_events:await self.bus.publish_outbound(OutboundMessage(session_key=session_key,- content=str(result),+ content=result_text,event_type=OutboundEventType.TOOL_RESULT,))+ model_result = result+ if isinstance(result, MultimodalToolResult):+ result_media_bytes = _inline_media_bytes([{"content": result.content}])+ if not self.provider.supports_tool_result_media(self.model):+ model_result = (+ f"{result_text}\n\nMedia content was omitted because the configured "+ "model provider does not support media in tool results."+ )+ else:+ overflow = (+ request_media_bytes+ + result_media_bytes+ - MAX_INLINE_TOOL_RESULT_MEDIA_BYTES+ )+ if overflow > 0:+ reclaimed = _demote_historical_tool_media(+ messages,+ history_end=history_end,+ bytes_needed=overflow,+ )+ request_media_bytes -= reclaimed+ if (+ request_media_bytes + result_media_bytes+ > MAX_INLINE_TOOL_RESULT_MEDIA_BYTES+ ):+ model_result = (+ f"{result_text}\n\nMedia content was omitted because adding it would "+ "make this model request exceed the inline media "+ f"limit of {MAX_INLINE_TOOL_RESULT_MEDIA_BYTES} bytes."+ )+ else:+ request_media_bytes += result_media_bytesmessages = self.context.add_tool_result(- messages, tool_call.id, tool_call.name, result+ messages, tool_call.id, tool_call.name, model_result)tool_used_dict = {@@ -1616,11 +1725,11 @@ class AgentLoop:"tool_name": tool_call.name,"args": args_str,"resolved_args": outcome.effective_params,- "result": result,+ "result": recorded_result,"duration": tool_execute_duration,"execute_success": _is_tool_result_success(result),"input_token": tool_call.tokens,- "output_token": cal_str_tokens(result, text_type="mixed"),+ "output_token": cal_str_tokens(result_text, text_type="mixed"),}if outcome.skill_uris:tool_used_dict["skill_uri"] = outcome.skill_uris[0]diff --git a/bot/vikingbot/agent/tools/base.py b/bot/vikingbot/agent/tools/base.pyindex 36a47a8a0..17bc9def6 100644--- a/bot/vikingbot/agent/tools/base.py+++ b/bot/vikingbot/agent/tools/base.py@@ -8,6 +8,17 @@ from vikingbot.config.schema import SessionKeyfrom vikingbot.sandbox.manager import SandboxManager+@dataclass(frozen=True)+class MultimodalToolResult:+ """A tool result with model-facing content and a text-only representation."""++ text: str+ content: list[dict[str, Any]]++ def __str__(self) -> str:+ return self.text++@dataclassclass ToolContext:"""Context passed to tools during execution, containing runtime information.@@ -152,7 +163,7 @@ class Tool(ABC):return {}@abstractmethod- async def execute(self, tool_context: ToolContext, **kwargs: Any) -> str:+ async def execute(self, tool_context: ToolContext, **kwargs: Any) -> str | MultimodalToolResult:"""Execute the tool with given parameters.@@ -161,7 +172,7 @@ class Tool(ABC):**kwargs: Tool-specific parameters.Returns:- String result of the tool execution.+ Text result, optionally with model-facing multimodal content."""passdiff --git a/bot/vikingbot/agent/tools/ov_file.py b/bot/vikingbot/agent/tools/ov_file.pyindex 398c7999c..f05d862ab 100644--- a/bot/vikingbot/agent/tools/ov_file.py+++ b/bot/vikingbot/agent/tools/ov_file.py@@ -1,19 +1,22 @@"""OpenViking file system tools: read, write, list, search resources."""import asyncio+import base64import itertoolsimport jsonimport tempfileimport timefrom abc import ABC-from pathlib import Path+from pathlib import Path, PurePosixPathfrom typing import TYPE_CHECKING, Any, Mapping, Optionalimport httpxfrom loguru import logger-from vikingbot.agent.tools.base import Tool, ToolContext+from openviking.utils.media_limits import MAX_INLINE_TOOL_RESULT_MEDIA_BYTES+from vikingbot.agent.tools.base import MultimodalToolResult, Tool, ToolContextfrom vikingbot.openviking_mount.ov_server import VikingClient+from vikingbot.utils.image_format import sniff_image_formatif TYPE_CHECKING:from vikingbot.config.schema import Config@@ -919,6 +922,30 @@ class VikingMultiReadTool(OVFileTool):"""Tool to read content from multiple Viking resources concurrently."""_FULL_READ_WARN_BYTES = 512 * 1024+ _MAX_INLINE_IMAGES = 4+ _MAX_INLINE_MEDIA_BYTES = MAX_INLINE_TOOL_RESULT_MEDIA_BYTES+ _INLINE_IMAGE_EXTENSIONS = frozenset({".gif", ".jpeg", ".jpg", ".png", ".webp"})+ _INLINE_IMAGE_MIME_TYPES = frozenset({"image/gif", "image/jpeg", "image/png", "image/webp"})+ _AUDIO_EXTENSIONS = frozenset(+ {".aac", ".ac3", ".flac", ".m4a", ".mp3", ".ogg", ".opus", ".wav"}+ )+ _VIDEO_EXTENSIONS = frozenset({".avi", ".flv", ".mkv", ".mov", ".mp4", ".webm", ".wmv"})++ @staticmethod+ def _uri_suffix(uri: str) -> str:+ path = uri.split("#", 1)[0].split("?", 1)[0]+ return PurePosixPath(path).suffix.lower()++ @classmethod+ def _media_kind(cls, uri: str) -> str | None:+ suffix = cls._uri_suffix(uri)+ if suffix in cls._INLINE_IMAGE_EXTENSIONS:+ return "image"+ if suffix in cls._AUDIO_EXTENSIONS:+ return "audio"+ if suffix in cls._VIDEO_EXTENSIONS:+ return "video"+ return None@propertydef name(self) -> str:@@ -964,7 +991,7 @@ class VikingMultiReadTool(OVFileTool):offset: int = 0,limit: int = -1,**kwargs: Any,- ) -> str:+ ) -> str | MultimodalToolResult:level = "read" # 默认获取完整内容client = Nonetry:@@ -975,9 +1002,104 @@ class VikingMultiReadTool(OVFileTool):max_concurrent = 10semaphore = asyncio.Semaphore(max_concurrent)- async def read_single_uri(uri: str) -> dict:+ image_indexes = [+ index for index, uri in enumerate(uris) if self._media_kind(uri) == "image"+ ]+ inline_image_indexes = set(image_indexes[: self._MAX_INLINE_IMAGES])++ async def read_single_uri(index: int, uri: str) -> dict[str, Any]:async with semaphore:try:+ media_kind = self._media_kind(uri)+ if media_kind == "image":+ if index not in inline_image_indexes:+ return {+ "uri": uri,+ "content": (+ "Image was not attached because one tool result supports "+ f"at most {self._MAX_INLINE_IMAGES} images. Read fewer "+ "images at once."+ ),+ "success": False,+ "media_kind": media_kind,+ }+ stat = await client.stat(uri)+ size = stat.get("size") if isinstance(stat, dict) else None+ if isinstance(size, bool) or not isinstance(size, int) or size < 0:+ return {+ "uri": uri,+ "content": "Image size is unavailable; cannot attach it safely.",+ "success": False,+ "media_kind": media_kind,+ }+ if size > self._MAX_INLINE_MEDIA_BYTES:+ return {+ "uri": uri,+ "content": (+ f"Image is {size} bytes; the inline limit is "+ f"{self._MAX_INLINE_MEDIA_BYTES} bytes."+ ),+ "success": False,+ "media_kind": media_kind,+ }+ data = await client.download_bytes(uri)+ if len(data) > size:+ return {+ "uri": uri,+ "content": "Image changed after its size was checked; retry the read.",+ "success": False,+ "media_kind": media_kind,+ }+ image_format = sniff_image_format(data)+ if (+ image_format is None+ or image_format.mime_type not in self._INLINE_IMAGE_MIME_TYPES+ ):+ return {+ "uri": uri,+ "content": (+ "File bytes do not match a supported PNG, JPEG, GIF, "+ "or WebP image."+ ),+ "success": False,+ "media_kind": media_kind,+ }+ encoded = base64.b64encode(data).decode("ascii")+ return {+ "uri": uri,+ "content": (+ f"Image resource ({image_format.mime_type}, {len(data)} bytes)."+ ),+ "image": {+ "type": "image_url",+ "image_url": {+ "url": f"data:{image_format.mime_type};base64,{encoded}"+ },+ },+ "media_bytes": len(data),+ "success": True,+ "media_kind": media_kind,+ }++ if media_kind in {"audio", "video"}:+ content = await client.read_content(uri, level="overview")+ if not content:+ return {+ "uri": uri,+ "content": (+ "No OpenViking textual overview is available for this "+ f"{media_kind} resource."+ ),+ "success": False,+ "media_kind": media_kind,+ }+ return {+ "uri": uri,+ "content": content,+ "success": True,+ "media_kind": media_kind,+ }+if limit == -1:try:stat = await client.stat(uri)@@ -994,6 +1116,7 @@ class VikingMultiReadTool(OVFileTool):"openviking_multi_read offset/limit."),"success": False,+ "media_kind": None,}content = await client.read_content(uri, level=level, offset=offset, limit=limit@@ -1007,6 +1130,7 @@ class VikingMultiReadTool(OVFileTool):"uri": uri,"content": content,"success": True,+ "media_kind": None,}except Exception as e:logger.warning(f"Error reading from {uri}: {e}")@@ -1014,12 +1138,29 @@ class VikingMultiReadTool(OVFileTool):"uri": uri,"content": f"Error reading from Viking: {str(e)}","success": False,+ "media_kind": self._media_kind(uri),}# 并发读取所有URI- read_tasks = [read_single_uri(uri) for uri in uris]+ read_tasks = [read_single_uri(index, uri) for index, uri in enumerate(uris)]results = await asyncio.gather(*read_tasks)+ inline_media_bytes = 0+ for result in results:+ if result.get("image") is None:+ continue+ media_bytes = result.get("media_bytes", 0)+ if inline_media_bytes + media_bytes > self._MAX_INLINE_MEDIA_BYTES:+ result.pop("image", None)+ result["media_bytes"] = 0+ result["content"] = (+ "Image was not attached because the combined inline media size would "+ f"exceed {self._MAX_INLINE_MEDIA_BYTES} bytes. Read fewer images at once."+ )+ result["success"] = False+ continue+ inline_media_bytes += media_bytes+# 构建结果range_note = (""@@ -1027,20 +1168,45 @@ class VikingMultiReadTool(OVFileTool):else f" (lines {offset}..{offset + limit - 1 if limit > 0 else 'end'})")result_lines = [f"Multi-read results for {len(uris)} resources (level: {level}):"]+ has_images = any(result.get("image") is not None for result in results)+ content_parts: list[dict[str, Any]] = (+ [{"type": "text", "text": result_lines[0]}] if has_images else []+ )for result in results:uri = result["uri"]content = result["content"]success = result["success"]+ item_range_note = "" if result["media_kind"] else range_note- result_lines.append(f"\n--- START OF {uri}{range_note} ---")+ start_marker = f"\n--- START OF {uri}{item_range_note} ---"+ end_marker = f"--- END OF {uri} ---"+ result_lines.append(start_marker)if success:result_lines.append(content)else:result_lines.append(f"ERROR: {content}")- result_lines.append(f"--- END OF {uri} ---")+ result_lines.append(end_marker)- return "\n".join(result_lines)+ if has_images:+ content_parts.append(+ {+ "type": "text",+ "text": f"{start_marker}\n{'ERROR: ' if not success else ''}{content}",+ }+ )+ image = result.get("image")+ if image is not None:+ content_parts.append(image)+ content_parts.append({"type": "text", "text": end_marker})++ text_result = "\n".join(result_lines)+ if has_images:+ return MultimodalToolResult(+ text=text_result,+ content=content_parts,+ )+ return text_resultexcept Exception as e:logger.exception(f"Error in VikingMultiReadTool: {e}")@@ -1178,9 +1344,7 @@ class VikingExportTool(OVFileTool):f"exceed the total-byte budget ({self._MAX_TOTAL_BYTES} bytes).")return f"No text files exported from {uri}."- lines = [- f"Exported {len(exported)} file(s) into the workspace under '{dest}/':"- ]+ lines = [f"Exported {len(exported)} file(s) into the workspace under '{dest}/':"]lines.extend(f"- {line}" for line in exported)if skipped_binary:lines.append(f"Skipped {skipped_binary} non-UTF-8 (binary) file(s).")diff --git a/bot/vikingbot/agent/tools/registry.py b/bot/vikingbot/agent/tools/registry.pyindex 395495bdf..debe0dbf0 100644--- a/bot/vikingbot/agent/tools/registry.py+++ b/bot/vikingbot/agent/tools/registry.py@@ -8,7 +8,7 @@ from typing import Anyfrom loguru import loggerfrom vikingbot.agent.remote_skills import SkillRuntimeError-from vikingbot.agent.tools.base import Tool, ToolContext+from vikingbot.agent.tools.base import MultimodalToolResult, Tool, ToolContextfrom vikingbot.config.schema import SessionKeyfrom vikingbot.hooks import HookContextfrom vikingbot.hooks.manager import hook_manager@@ -340,6 +340,8 @@ class ToolRegistry:channel_metadata=channel_metadata,skill_runtime=skill_runtime,)+ if isinstance(outcome.result, MultimodalToolResult):+ return str(outcome.result)return outcome.result@propertydiff --git a/bot/vikingbot/hooks/builtins/openviking_hooks.py b/bot/vikingbot/hooks/builtins/openviking_hooks.pyindex ea40f4228..1c69502ca 100644--- a/bot/vikingbot/hooks/builtins/openviking_hooks.py+++ b/bot/vikingbot/hooks/builtins/openviking_hooks.py@@ -409,12 +409,13 @@ class OpenVikingPostCallHook(Hook):)if is_skill_read:if result and not isinstance(result, Exception):- match = re.search(r"^---\s*\nname:\s*(.+?)\s*\n", result, re.MULTILINE)+ result_text = str(result)+ match = re.search(r"^---\s*\nname:\s*(.+?)\s*\n", result_text, re.MULTILINE)if match:skill_name = match.group(1).strip()if skill_name == "experience_loader":return {"tool_name": tool_name, "params": params, "result": result}- desc_match = re.search(r"^description:\s*(.+)$", result, re.MULTILINE)+ desc_match = re.search(r"^description:\s*(.+)$", result_text, re.MULTILINE)skill_query = desc_match.group(1).strip() if desc_match else skill_nameexp_memory = await self._search_skill_experiences(@@ -424,7 +425,7 @@ class OpenVikingPostCallHook(Hook):openviking_connection=context.openviking_connection,)if exp_memory:- result = f"{result}\n\n---\n## Related Experiences\n{exp_memory}"+ result = f"{result_text}\n\n---\n## Related Experiences\n{exp_memory}"return {"tool_name": tool_name, "params": params, "result": result}diff --git a/bot/vikingbot/openviking_mount/ov_server.py b/bot/vikingbot/openviking_mount/ov_server.pyindex 14a6fccee..6aa9a113c 100644--- a/bot/vikingbot/openviking_mount/ov_server.py+++ b/bot/vikingbot/openviking_mount/ov_server.py@@ -597,7 +597,17 @@ class VikingClient:return entriesasync def stat(self, uri: str) -> Dict[str, Any]:- return await self.client.stat(uri)+ client = self.client+ should_close = False+ scoped_user_id = self._owner_user_id_for_uri(uri)+ if scoped_user_id:+ client, should_close = await self._get_user_scoped_client(scoped_user_id)++ try:+ return await client.stat(uri)+ finally:+ if should_close:+ await client.close()async def attrs(self, uri: str) -> Dict[str, Any]:return await self.client.attrs(uri)@@ -612,7 +622,17 @@ class VikingClient:return await self.client.read_raw(uri, offset=offset, limit=limit)async def download_bytes(self, uri: str) -> bytes:- return await self.client.download_bytes(uri)+ client = self.client+ should_close = False+ scoped_user_id = self._owner_user_id_for_uri(uri)+ if scoped_user_id:+ client, should_close = await self._get_user_scoped_client(scoped_user_id)++ try:+ return await client.download_bytes(uri)+ finally:+ if should_close:+ await client.close()async def find_skills(self,diff --git a/bot/vikingbot/providers/base.py b/bot/vikingbot/providers/base.pyindex 7b1d16591..183d5660f 100644--- a/bot/vikingbot/providers/base.py+++ b/bot/vikingbot/providers/base.py@@ -192,6 +192,10 @@ class LLMProvider(ABC):)yield LLMStreamEvent(type="response", response=response)+ def supports_tool_result_media(self, model: str | None = None) -> bool:+ """Whether this provider can preserve media inside a tool result."""+ return False+@abstractmethoddef get_default_model(self) -> str:"""Get the default model for this provider."""diff --git a/bot/vikingbot/providers/litellm_provider.py b/bot/vikingbot/providers/litellm_provider.pyindex 74d7d155b..70f07f411 100644--- a/bot/vikingbot/providers/litellm_provider.py+++ b/bot/vikingbot/providers/litellm_provider.py@@ -563,3 +563,10 @@ class LiteLLMProvider(LLMProvider):def get_default_model(self) -> str:"""Get the default model."""return self.default_model++ def supports_tool_result_media(self, model: str | None = None) -> bool:+ """Enable only the native tool-result media path verified for Anthropic."""+ if self._gateway is not None:+ return False+ spec = find_by_model(model or self.default_model)+ return spec is not None and spec.name == "anthropic"diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.pyindex 9787bf491..7600a2d57 100644--- a/bot/vikingbot/providers/vlm_adapter.py+++ b/bot/vikingbot/providers/vlm_adapter.py@@ -721,6 +721,32 @@ class VLMProviderAdapter(LLMProvider):def get_default_model(self) -> str:return self._default_model+ def supports_tool_result_media(self, model: str | None = None) -> bool:+ """Return true only for backends with native rich tool-result support."""+ return self._backend_supports_tool_result_media(self._vlm)++ @classmethod+ def _backend_supports_tool_result_media(cls, vlm: Any) -> bool:+ instances = getattr(vlm, "_vlm_instances", None)+ if instances is not None:+ return bool(instances) and all(+ cls._backend_supports_tool_result_media(item) for item in instances+ )++ primary = getattr(vlm, "primary", None)+ backup = getattr(vlm, "backup", None)+ if primary is not None and backup is not None:+ return cls._backend_supports_tool_result_media(+ primary+ ) and cls._backend_supports_tool_result_media(backup)++ resolve_provider = getattr(vlm, "resolved_provider", None)+ provider = (+ resolve_provider() if callable(resolve_provider) else getattr(vlm, "provider", "")+ )+ provider = str(provider or "").lower()+ return provider in {"anthropic", "openai-codex"}+# ------------------------------------------------------------------# Langfuse helpers (same pattern as LiteLLMProvider.chat())# ------------------------------------------------------------------diff --git a/bot/vikingbot/tests/unit/test_bot_provider_thinking.py b/bot/vikingbot/tests/unit/test_bot_provider_thinking.pyindex ae61f4315..d05a4c36f 100644--- a/bot/vikingbot/tests/unit/test_bot_provider_thinking.py+++ b/bot/vikingbot/tests/unit/test_bot_provider_thinking.py@@ -4,6 +4,9 @@ from types import SimpleNamespaceimport pytest+from openviking.models.vlm.backends.litellm_vlm import (+ LiteLLMVLMProvider as OpenVikingLiteLLMVLMProvider,+)from vikingbot.config.schema import AgentsConfigfrom vikingbot.providers.litellm_provider import LiteLLMProviderfrom vikingbot.providers.vlm_adapter import VLMProviderAdapter@@ -29,6 +32,73 @@ def test_agents_openviking_retention_defaults_to_turn_budget_values():assert config.commit_min_raw_tail_steps == 1+def test_vlm_adapter_exposes_only_native_tool_result_media_backends():+ langfuse = SimpleNamespace()++ codex = VLMProviderAdapter(+ SimpleNamespace(provider="openai-codex"),+ default_model="gpt-5.3-codex",+ langfuse_client=langfuse,+ )+ anthropic = VLMProviderAdapter(+ SimpleNamespace(provider="anthropic"),+ default_model="claude-sonnet",+ langfuse_client=langfuse,+ )+ openai = VLMProviderAdapter(+ SimpleNamespace(provider="openai"),+ default_model="gpt-4o",+ langfuse_client=langfuse,+ )++ assert codex.supports_tool_result_media() is True+ assert anthropic.supports_tool_result_media() is True+ assert openai.supports_tool_result_media() is False++ mixed_failover = VLMProviderAdapter(+ SimpleNamespace(+ provider="anthropic",+ primary=SimpleNamespace(provider="anthropic"),+ backup=SimpleNamespace(provider="openai"),+ ),+ default_model="claude-sonnet",+ langfuse_client=langfuse,+ )+ assert mixed_failover.supports_tool_result_media() is False+++def test_litellm_provider_exposes_anthropic_tool_result_media_only():+ langfuse = SimpleNamespace()+ anthropic = LiteLLMProvider(+ default_model="claude-sonnet-4-5",+ langfuse_client=langfuse,+ )+ openai = LiteLLMProvider(+ default_model="gpt-4o",+ langfuse_client=langfuse,+ )++ assert anthropic.supports_tool_result_media() is True+ assert openai.supports_tool_result_media() is False+++def test_vlm_adapter_uses_litellm_resolved_provider_for_tool_result_media():+ vlm = OpenVikingLiteLLMVLMProvider(+ {+ "provider": "litellm",+ "model": "claude-sonnet-4-5",+ }+ )+ provider = VLMProviderAdapter(+ vlm,+ default_model="claude-sonnet-4-5",+ langfuse_client=SimpleNamespace(),+ )++ assert vlm.resolved_provider() == "anthropic"+ assert provider.supports_tool_result_media() is True++def test_make_provider_passes_default_thinking_to_vlm_adapter(monkeypatch):monkeypatch.setitem(sys.modules,diff --git a/bot/vikingbot/utils/image_format.py b/bot/vikingbot/utils/image_format.pyindex 8ec7bfff5..08ea6978e 100644--- a/bot/vikingbot/utils/image_format.py+++ b/bot/vikingbot/utils/image_format.py@@ -27,8 +27,8 @@ def image_format_from_mime(mime_type: str | None) -> ImageFormat | None:return _MIME_FORMATS.get(mime_type.lower().split(";", 1)[0].strip())-def detect_image_format(image_data: bytes, fallback_mime: str | None = None) -> ImageFormat:- """Detect common image formats from magic bytes, falling back to a MIME hint."""+def sniff_image_format(image_data: bytes) -> ImageFormat | None:+ """Detect common image formats from magic bytes."""if image_data.startswith(b"\x89PNG\r\n\x1a\n"):return PNG_FORMATif image_data.startswith(b"\xff\xd8\xff"):@@ -40,4 +40,13 @@ def detect_image_format(image_data: bytes, fallback_mime: str | None = None) ->if len(image_data) >= 12 and image_data[:4] == b"RIFF" and image_data[8:12] == b"WEBP":return ImageFormat("webp", "image/webp")+ return None+++def detect_image_format(image_data: bytes, fallback_mime: str | None = None) -> ImageFormat:+ """Detect common image formats from magic bytes, falling back to a MIME hint."""+ detected = sniff_image_format(image_data)+ if detected is not None:+ return detected+return image_format_from_mime(fallback_mime) or PNG_FORMATdiff --git a/openviking/models/vlm/backends/codex_responses_adapter.py b/openviking/models/vlm/backends/codex_responses_adapter.pyindex d1bf0e3a7..f650de40c 100644--- a/openviking/models/vlm/backends/codex_responses_adapter.py+++ b/openviking/models/vlm/backends/codex_responses_adapter.py@@ -86,6 +86,16 @@ def _stringify_response_payload(value: Any, *, default: str = "") -> str:return str(value)+def _convert_tool_output_for_responses(content: Any) -> Any:+ if isinstance(content, list):+ converted = _convert_content_for_responses(content)+ if isinstance(converted, list) and any(+ part.get("type") == "input_image" for part in converted if isinstance(part, dict)+ ):+ return converted+ return _stringify_response_payload(content)++def _convert_tool_call_history(tool_calls: Any) -> List[Dict[str, Any]]:if not isinstance(tool_calls, list):return []@@ -118,7 +128,7 @@ def _convert_message_for_responses(message: Dict[str, Any]) -> List[Dict[str, An{"type": "function_call_output","call_id": str(message.get("tool_call_id", "") or message.get("call_id", "") or ""),- "output": _stringify_response_payload(content),+ "output": _convert_tool_output_for_responses(content),}]if role == "assistant":diff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.pyindex cbd7c56e0..84e9057a8 100644--- a/openviking/models/vlm/backends/litellm_vlm.py+++ b/openviking/models/vlm/backends/litellm_vlm.py@@ -211,6 +211,13 @@ class LiteLLMVLMProvider(VLMBase):return model+ def resolved_provider(self, model: str | None = None) -> str | None:+ """Return the provider LiteLLM will route this model to."""+ resolved_model = self._resolve_model(model or self.model or "")+ if "/" in resolved_model:+ return resolved_model.split("/", 1)[0].lower()+ return self._detected_provider or detect_provider_by_model(resolved_model) or self.provider+def _should_forward_api_key(self, model: str) -> bool:if not self.api_key:return Falsediff --git a/openviking/server/mcp_endpoint.py b/openviking/server/mcp_endpoint.pyindex 45646dc55..c732a8b3f 100644--- a/openviking/server/mcp_endpoint.py+++ b/openviking/server/mcp_endpoint.py@@ -65,6 +65,7 @@ from openviking.server.local_input_guard import (from openviking.server.resource_ingest import ingest_temp_uploadfrom openviking.server.temp_upload_store import TempUploadStorefrom openviking.server.upload_token_store import upload_token_store+from openviking.utils.media_limits import MAX_INLINE_TOOL_RESULT_MEDIA_BYTESfrom openviking.utils.search_filters import SearchContextTypeInput, merge_search_filterfrom openviking_cli.exceptions import (InvalidArgumentError,@@ -425,9 +426,7 @@ async def _format_search_result(result, *, service, ctx, read_content: bool = Fa_MCP_IMAGE_EXTENSIONS = {".gif", ".jpeg", ".jpg", ".png", ".webp"}_MCP_AUDIO_EXTENSIONS = {".flac", ".m4a", ".mp3", ".oga", ".ogg", ".wav"}_MCP_VIDEO_EXTENSIONS = {".avi", ".m4v", ".mkv", ".mov", ".mp4", ".webm"}-# Common clients cap an inline result at 5 MiB base64, or 3.75 MiB raw.-# Apply the same limit to one file and to the aggregate media in one tool call.-_MCP_MEDIA_MAX_BYTES = 3_932_160+_MCP_MEDIA_MAX_BYTES = MAX_INLINE_TOOL_RESULT_MEDIA_BYTESdef _mcp_uri_suffix(uri: str) -> str:diff --git a/openviking/utils/media_limits.py b/openviking/utils/media_limits.pyindex c64fbba3d..dff555fc0 100644--- a/openviking/utils/media_limits.py+++ b/openviking/utils/media_limits.py@@ -5,6 +5,9 @@from typing import ProtocolMAX_MEDIA_FILE_BYTES = 512 * 1024 * 1024+# Common model clients cap one inline tool result at 5 MiB after Base64 encoding.+# Apply the equivalent raw-byte budget to both individual media and aggregates.+MAX_INLINE_TOOL_RESULT_MEDIA_BYTES = 3_932_160DEFAULT_LARGE_IMAGE_MAX_FILE_SIZE_MB = 10.0DEFAULT_LARGE_IMAGE_THRESHOLD_DIMENSION = 4096diff --git a/tests/unit/test_codex_vlm.py b/tests/unit/test_codex_vlm.pyindex 10d40987b..664b7012f 100644--- a/tests/unit/test_codex_vlm.py+++ b/tests/unit/test_codex_vlm.py@@ -10,6 +10,9 @@ import pytestfrom openviking.models.vlm.backends import codex_authfrom openviking.models.vlm.backends.codex_auth import resolve_codex_runtime_credentials+from openviking.models.vlm.backends.codex_responses_adapter import (+ _convert_message_for_responses,+)from openviking.models.vlm.backends.codex_vlm import CodexVLMfrom openviking_cli.utils.config.vlm_config import VLMConfig@@ -568,6 +571,89 @@ def test_codex_translates_tool_history_into_responses_input(mock_resolve, mock_o]+ ("content", "expected_output"),+ [+ (+ [+ {"type": "text", "text": "first"},+ {"type": "text", "text": "second"},+ ],+ "firstsecond",+ ),+ ({"ok": True}, '{"ok": true}'),+ ],+)+def test_codex_preserves_legacy_text_tool_output_serialization(content, expected_output):+ converted = _convert_message_for_responses(+ {+ "role": "tool",+ "tool_call_id": "call_1",+ "content": content,+ }+ )++ assert converted == [+ {+ "type": "function_call_output",+ "call_id": "call_1",+ "output": expected_output,+ }+ ]+++@patch("openviking.models.vlm.backends.codex_vlm.openai.OpenAI")+@patch("openviking.models.vlm.backends.codex_vlm.resolve_codex_runtime_credentials")+def test_codex_preserves_images_in_function_call_output(mock_resolve, mock_openai_class):+ mock_resolve.return_value = {+ "api_key": "oauth-token",+ "base_url": "https://chatgpt.com/backend-api/codex",+ }+ mock_real_client = MagicMock()+ mock_real_client.responses.create.return_value = _MockResponsesStream(+ _build_final_response("image inspected")+ )+ mock_openai_class.return_value = mock_real_client+ image_url = "data:image/png;base64,aW1hZ2U="++ vlm = CodexVLM({"provider": "openai-codex", "model": "gpt-5.3-codex"})+ result = vlm.get_completion(+ messages=[+ {"role": "user", "content": "Inspect the image"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {+ "id": "call_1",+ "type": "function",+ "function": {"name": "read_image", "arguments": "{}"},+ }+ ],+ },+ {+ "role": "tool",+ "tool_call_id": "call_1",+ "content": [+ {"type": "text", "text": "Source: image.png"},+ {"type": "image_url", "image_url": {"url": image_url}},+ ],+ },+ ]+ )++ assert result == "image inspected"+ input_items = mock_real_client.responses.create.call_args.kwargs["input"]+ assert input_items[-1] == {+ "type": "function_call_output",+ "call_id": "call_1",+ "output": [+ {"type": "input_text", "text": "Source: image.png"},+ {"type": "input_image", "image_url": image_url},+ ],+ }++def test_codex_auth_invalid_exp_claim_is_treated_as_expiring():assert codex_auth._codex_access_token_is_expiring("not-a-jwt", skew_seconds=60) is Trueassert (