Skip to content

Agent Zero AI framework

Fix MCP multimodal content handling

Alessandro4f06aa0a8eae
diff --git a/helpers/mcp_handler.py b/helpers/mcp_handler.py
index 439127b21..ea23d61f4 100644
--- a/helpers/mcp_handler.py
+++ b/helpers/mcp_handler.py
@@ -21,6 +21,7 @@ from contextlib import AsyncExitStack
from shutil import which
from datetime import timedelta
import json
+import uuid
from helpers import errors
from helpers import settings
from helpers.log import LogItem
@@ -39,11 +40,21 @@ from anyio.streams.memory import (
)
from pydantic import BaseModel, Field, Discriminator, Tag, PrivateAttr
-from helpers import dirty_json
+from helpers import dirty_json, media_artifacts
from helpers.print_style import PrintStyle
from helpers.tool import Tool, Response
+MCP_MEDIA_TOKENS_ESTIMATE = 1500
+MAX_MCP_RESOURCE_TEXT_CHARS = 12_000
+
+
+def _mcp_get(item: Any, key: str, default: Any = None) -> Any:
+ if isinstance(item, dict):
+ return item.get(key, default)
+ return getattr(item, key, default)
+
+
def normalize_name(name: str) -> str:
# Lowercase and strip whitespace
name = name.strip().lower()
@@ -102,7 +113,6 @@ class MCPTool(Tool):
"""MCP Tool wrapper"""
def get_log_object(self) -> LogItem:
- import uuid
return self.agent.context.log.log(
type="mcp",
heading=f"icon://extension {self.agent.agent_name}: Using MCP tool '{self.name}'",
@@ -111,17 +121,235 @@ class MCPTool(Tool):
id=str(uuid.uuid4()),
)
+ def _context_id(self) -> str:
+ return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()
+
+ def _raw_tool_response(self, response: Response) -> str:
+ raw_tool_response = response.message.strip() if response.message else ""
+ if not raw_tool_response:
+ PrintStyle(font_color="red").print(
+ f"Warning: Tool '{self.name}' returned an empty message."
+ )
+ raw_tool_response = "[Tool returned no textual content]"
+ return raw_tool_response
+
+ def _coerce_media_token_estimate(self, value: object) -> int:
+ try:
+ estimate = int(value or 0)
+ except (TypeError, ValueError):
+ estimate = 0
+ return estimate if estimate > 0 else MCP_MEDIA_TOKENS_ESTIMATE
+
+ def _format_image_content(
+ self,
+ *,
+ encoded: str,
+ mime_type: str,
+ label: str,
+ ) -> tuple[str, dict[str, Any] | None]:
+ try:
+ image = media_artifacts.image_data_url_from_base64(
+ encoded,
+ mime_type=mime_type,
+ )
+ except media_artifacts.EmptyBase64Data:
+ return f"MCP returned an empty {label} attachment.", None
+ except media_artifacts.InvalidBase64Data:
+ return f"MCP returned a {label} attachment that could not be decoded.", None
+
+ return (
+ f"MCP returned {label} attachment ({image.mime}, {image.size} bytes).",
+ {
+ "type": "image_url",
+ "image_url": {"url": image.url},
+ },
+ )
+
+ def _materialize_binary_content(
+ self,
+ *,
+ encoded: str,
+ mime_type: str,
+ label: str,
+ index: int,
+ preferred_name: str = "",
+ ) -> str:
+ try:
+ safe_mime = media_artifacts.normalize_mime(mime_type)
+ artifact = media_artifacts.save_base64_artifact(
+ encoded,
+ mime_type=safe_mime,
+ directory_parts=self._artifact_directory_parts(),
+ preferred_name=preferred_name,
+ default_filename=self._default_artifact_filename(
+ label=label,
+ index=index,
+ mime_type=safe_mime,
+ ),
+ )
+ except media_artifacts.EmptyBase64Data:
+ return f"MCP returned an empty {label} attachment."
+ except media_artifacts.InvalidBase64Data:
+ return f"MCP returned a {label} attachment that could not be decoded."
+
+ return f"Saved MCP {label} attachment ({artifact.mime}, {artifact.size} bytes) to {artifact.path}."
+
+ def _artifact_directory_parts(self) -> tuple[str, ...]:
+ context_id = normalize_name(self._context_id() or "shared") or "shared"
+ tool_name = normalize_name(self.name or "mcp_tool") or "mcp_tool"
+ return ("tmp", "mcp", context_id, tool_name)
+
+ def _default_artifact_filename(self, *, label: str, index: int, mime_type: str) -> str:
+ tool_name = normalize_name(self.name or "mcp_tool") or "mcp_tool"
+ ext = media_artifacts.guess_extension(mime_type, ".bin")
+ return f"{tool_name}_{label}_{index}{ext}"
+
+ def _format_resource_text(self, text: str, uri: str = "") -> str:
+ body = str(text or "").strip()
+ if not body:
+ return ""
+ if len(body) > MAX_MCP_RESOURCE_TEXT_CHARS:
+ body = body[:MAX_MCP_RESOURCE_TEXT_CHARS].rstrip() + "\n...[truncated]"
+ if uri:
+ return f"Resource {uri}:\n{body}"
+ return body
+
+ def _content_item_dump(self, item: Any) -> dict[str, Any]:
+ if isinstance(item, dict):
+ return dict(item)
+ model_dump = getattr(item, "model_dump", None)
+ if callable(model_dump):
+ dumped = model_dump(mode="python")
+ if isinstance(dumped, dict):
+ return dumped
+ item_vars = getattr(item, "__dict__", None)
+ if isinstance(item_vars, dict):
+ return dict(item_vars)
+ return {}
+
+ def _summarize_unknown_item(self, item: Any, item_type: str) -> str:
+ dumped = self._content_item_dump(item)
+ if dumped:
+ dumped.pop("data", None)
+ resource = dumped.get("resource")
+ if isinstance(resource, dict):
+ resource.pop("blob", None)
+ summary = json.dumps(dumped, ensure_ascii=False)
+ if len(summary) > 600:
+ summary = summary[:600] + "...[truncated]"
+ return f"MCP returned unsupported content item type '{item_type}': {summary}"
+ return f"MCP returned unsupported content item type '{item_type}'."
+
+ def _format_tool_result(
+ self, response: CallToolResult
+ ) -> tuple[str, dict[str, Any] | None]:
+ text_parts: list[str] = []
+ notes: list[str] = []
+ raw_images: list[dict[str, Any]] = []
+ content_items = list(getattr(response, "content", []) or [])
+
+ for index, item in enumerate(content_items, start=1):
+ item_type = str(_mcp_get(item, "type", "") or "").strip().lower()
+
+ if item_type == "text":
+ text = str(_mcp_get(item, "text", "") or "").strip()
+ if text:
+ text_parts.append(text)
+ continue
+
+ if item_type == "image":
+ note, raw_content = self._format_image_content(
+ encoded=str(_mcp_get(item, "data", "") or ""),
+ mime_type=str(_mcp_get(item, "mimeType", "") or "image/png"),
+ label="image",
+ )
+ notes.append(note)
+ if raw_content:
+ raw_images.append(raw_content)
+ continue
+
+ if item_type == "audio":
+ note = self._materialize_binary_content(
+ encoded=str(_mcp_get(item, "data", "") or ""),
+ mime_type=str(_mcp_get(item, "mimeType", "") or "audio/wav"),
+ label="audio",
+ index=index,
+ )
+ notes.append(note)
+ continue
+
+ if item_type == "resource":
+ resource = _mcp_get(item, "resource", None)
+ uri = str(_mcp_get(resource, "uri", "") or "").strip()
+ text = _mcp_get(resource, "text", None)
+ if isinstance(text, str) and text.strip():
+ text_parts.append(self._format_resource_text(text, uri))
+ continue
+
+ blob = str(_mcp_get(resource, "blob", "") or "").strip()
+ if blob:
+ mime_type = str(
+ _mcp_get(resource, "mimeType", "") or "application/octet-stream"
+ ).strip().lower()
+ if mime_type.startswith("image/"):
+ note, raw_content = self._format_image_content(
+ encoded=blob,
+ mime_type=mime_type,
+ label="resource image",
+ )
+ else:
+ note = self._materialize_binary_content(
+ encoded=blob,
+ mime_type=mime_type,
+ label="resource",
+ index=index,
+ preferred_name=uri,
+ )
+ raw_content = None
+ notes.append(note)
+ if raw_content:
+ raw_images.append(raw_content)
+ continue
+
+ if uri:
+ mime_type = str(_mcp_get(resource, "mimeType", "") or "").strip()
+ details = f" ({mime_type})" if mime_type else ""
+ notes.append(f"MCP returned a resource reference: {uri}{details}.")
+ continue
+
+ notes.append("MCP returned a resource item without text or binary data.")
+ continue
+
+ if item_type:
+ notes.append(self._summarize_unknown_item(item, item_type))
+ continue
+
+ notes.append(self._summarize_unknown_item(item, "unknown"))
+
+ message = "\n\n".join(part for part in [*text_parts, *notes] if part.strip())
+ if not message and content_items:
+ message = "MCP tool returned content that could not be rendered as text."
+
+ additional = None
+ if raw_images:
+ additional = {
+ "raw_content": raw_images,
+ "preview": f"<MCP image attachments: {len(raw_images)}>",
+ "_tokens": MCP_MEDIA_TOKENS_ESTIMATE * len(raw_images),
+ }
+
+ return message, additional
+
async def execute(self, **kwargs: Any):
error = ""
+ additional: dict[str, Any] | None = None
try:
response: CallToolResult = await MCPConfig.get_instance().call_tool(
self.name, kwargs
)
- message = "\n\n".join(
- [item.text for item in response.content if item.type == "text"]
- )
+ message, additional = self._format_tool_result(response)
if response.isError:
- error = message
+ error = message or "MCP tool returned an error without textual content."
except Exception as e:
error = f"MCP Tool Exception: {str(e)}"
message = f"ERROR: {str(e)}"
@@ -139,7 +367,7 @@ class MCPTool(Tool):
content=f"{self.name}: {error}",
)
- return Response(message=message, break_loop=False)
+ return Response(message=message, break_loop=False, additional=additional)
async def before_execution(self, **kwargs: Any):
(
@@ -159,48 +387,29 @@ class MCPTool(Tool):
PrintStyle().print()
async def after_execution(self, response: Response, **kwargs: Any):
- raw_tool_response = response.message.strip() if response.message else ""
- if not raw_tool_response:
- PrintStyle(font_color="red").print(
- f"Warning: Tool '{self.name}' returned an empty message."
+ final_text_for_agent = self._raw_tool_response(response)
+ additional = dict(response.additional or {})
+ raw_content = additional.pop("raw_content", None)
+ preview = str(additional.pop("preview", "") or "").strip()
+ token_estimate = self._coerce_media_token_estimate(additional.pop("_tokens", 0))
+
+ self.agent.hist_add_tool_result(
+ self.name,
+ final_text_for_agent,
+ id=self.log.id if self.log else "",
+ **additional,
+ )
+ if raw_content:
+ from helpers import history
+
+ self.agent.hist_add_message(
+ False,
+ content=history.RawMessage(
+ raw_content=raw_content,
+ preview=preview or final_text_for_agent,
+ ),
+ tokens=token_estimate,
)
- # Even if empty, we might still want to provide context for the agent
- raw_tool_response = "[Tool returned no textual content]"
-
- # Prepare user message context
- # user_message_text = (
- # "No specific user message context available for this exact step."
- # )
- # if (
- # self.agent
- # and self.agent.last_user_message
- # and self.agent.last_user_message.content
- # ):
- # content = self.agent.last_user_message.content
- # if isinstance(content, dict):
- # # Attempt to get a 'message' field, otherwise stringify the dict
- # user_message_text = str(content.get(
- # "message", json.dumps(content, indent=2)
- # ))
- # elif isinstance(content, str):
- # user_message_text = content
- # else:
- # # Fallback for any other types (e.g. list, if that were possible for content)
- # user_message_text = str(content)
-
- # # Ensure user_message_text is a string before length check and slicing
- # user_message_text = str(user_message_text)
-
- # # Truncate user message context if it's too long to avoid overwhelming the prompt
- # max_user_context_len = 500 # characters
- # if len(user_message_text) > max_user_context_len:
- # user_message_text = (
- # user_message_text[:max_user_context_len] + "... (truncated)"
- # )
-
- final_text_for_agent = raw_tool_response
-
- self.agent.hist_add_tool_result(self.name, final_text_for_agent, id=self.log.id if self.log else "")
(
PrintStyle(
font_color="#1B4F72", background_color="white", padding=True, bold=True
@@ -210,8 +419,8 @@ class MCPTool(Tool):
)
# Print only the raw response to console for brevity, agent gets the full context.
PrintStyle(font_color="#85C1E9").print(
- raw_tool_response
- if raw_tool_response
+ final_text_for_agent
+ if final_text_for_agent
else "[No direct textual output from tool]"
)
if self.log:
diff --git a/helpers/media_artifacts.py b/helpers/media_artifacts.py
new file mode 100644
index 000000000..cfd89673e
--- /dev/null
+++ b/helpers/media_artifacts.py
@@ -0,0 +1,186 @@
+from __future__ import annotations
+
+import base64
+import binascii
+import mimetypes
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from urllib.parse import urlparse
+
+from helpers import files
+
+
+DEFAULT_MAX_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024
+
+
+class MediaArtifactError(ValueError):
+ pass
+
+
+class EmptyBase64Data(MediaArtifactError):
+ pass
+
+
+class InvalidBase64Data(MediaArtifactError):
+ pass
+
+
+class ArtifactTooLarge(MediaArtifactError):
+ def __init__(self, size: int, limit: int):
+ super().__init__(f"artifact is too large ({size} bytes, limit {limit} bytes)")
+ self.size = size
+ self.limit = limit
+
+
+@dataclass(frozen=True)
+class Base64Payload:
+ data: str
+ payload: bytes
+ size: int
+
+
+@dataclass(frozen=True)
+class ImageDataUrl:
+ url: str
+ mime: str
+ size: int
+
+
+@dataclass(frozen=True)
+class SavedArtifact:
+ path: str
+ mime: str
+ size: int
+
+
+def compact_base64(data: str) -> str:
+ return "".join(char for char in str(data or "") if not char.isspace())
+
+
+def estimated_base64_decoded_size(data: str) -> int:
+ compact_length = len(compact_base64(data))
+ return (compact_length * 3) // 4
+
+
+def decode_base64_payload(
+ data: str,
+ *,
+ max_bytes: int | None = None,
+) -> Base64Payload:
+ compact = compact_base64(data)
+ if not compact:
+ raise EmptyBase64Data("base64 data is empty")
+
+ if max_bytes is not None:
+ estimated_size = estimated_base64_decoded_size(compact)
+ if estimated_size > max_bytes:
+ raise ArtifactTooLarge(estimated_size, max_bytes)
+
+ try:
+ payload = base64.b64decode(compact, validate=True)
+ except (binascii.Error, ValueError) as exc:
+ raise InvalidBase64Data("base64 data could not be decoded") from exc
+
+ if max_bytes is not None and len(payload) > max_bytes:
+ raise ArtifactTooLarge(len(payload), max_bytes)
+
+ return Base64Payload(data=compact, payload=payload, size=len(payload))
+
+
+def normalize_mime(
+ mime_type: str,
+ *,
+ default: str = "application/octet-stream",
+ required_prefix: str = "",
+) -> str:
+ value = str(mime_type or "").strip().lower()
+ if not value:
+ return default
+ if required_prefix and not value.startswith(required_prefix):
+ return default
+ return value
+
+
+def guess_extension(mime_type: str, fallback: str = ".bin") -> str:
+ ext = mimetypes.guess_extension(str(mime_type or "").strip().lower()) or fallback
+ return ".jpg" if ext == ".jpe" else ext
+
+
+def filename_from_uri(uri: str) -> str:
+ value = str(uri or "").strip()
+ if not value:
+ return ""
+ parsed = urlparse(value)
+ return Path(parsed.path or value).name
+
+
+def safe_filename(
+ value: str,
+ *,
+ default: str = "artifact.bin",
+ default_extension: str = ".bin",
+) -> str:
+ source = str(value or "").strip() or default
+ cleaned = "".join(
+ char if char.isalnum() or char in {"-", "_", "."} else "_"
+ for char in source
+ )
+ cleaned = cleaned.strip("._") or default
+ if "." not in cleaned:
+ ext = default_extension if default_extension.startswith(".") else f".{default_extension}"
+ cleaned += ext
+ return cleaned
+
+
+def image_data_url_from_base64(
+ data: str,
+ *,
+ mime_type: str = "image/png",
+ max_bytes: int | None = None,
+) -> ImageDataUrl:
+ payload = decode_base64_payload(data, max_bytes=max_bytes)
+ safe_mime = normalize_mime(
+ mime_type,
+ default="image/png",
+ required_prefix="image/",
+ )
+ return ImageDataUrl(
+ url=f"data:{safe_mime};base64,{payload.data}",
+ mime=safe_mime,
+ size=payload.size,
+ )
+
+
+def save_base64_artifact(
+ data: str,
+ *,
+ mime_type: str = "application/octet-stream",
+ directory_parts: tuple[str, ...],
+ preferred_name: str = "",
+ default_filename: str = "artifact.bin",
+ max_bytes: int | None = None,
+) -> SavedArtifact:
+ payload = decode_base64_payload(data, max_bytes=max_bytes)
+ safe_mime = normalize_mime(mime_type)
+ preferred_filename = filename_from_uri(preferred_name) or default_filename
+ default_extension = guess_extension(safe_mime, Path(default_filename).suffix or ".bin")
+ filename = safe_filename(
+ preferred_filename,
+ default=default_filename,
+ default_extension=default_extension,
+ )
+ filename_path = Path(filename)
+ stem = filename_path.stem or Path(default_filename).stem or "artifact"
+ suffix = filename_path.suffix or default_extension
+
+ artifact_dir = Path(files.get_abs_path(*directory_parts))
+ artifact_dir.mkdir(parents=True, exist_ok=True)
+ path = artifact_dir / f"{stem}_{uuid.uuid4().hex[:8]}{suffix}"
+ path.write_bytes(payload.payload)
+
+ return SavedArtifact(
+ path=files.normalize_a0_path(str(path)),
+ mime=safe_mime,
+ size=payload.size,
+ )
diff --git a/plugins/_a0_connector/tools/computer_use_remote.py b/plugins/_a0_connector/tools/computer_use_remote.py
index 875ffec51..692c40b62 100644
--- a/plugins/_a0_connector/tools/computer_use_remote.py
+++ b/plugins/_a0_connector/tools/computer_use_remote.py
@@ -6,7 +6,7 @@ from pathlib import Path
import uuid
from typing import Any
-from helpers import history
+from helpers import history, media_artifacts
from helpers.print_style import PrintStyle
from helpers.tool import Response, Tool
from helpers.ws import NAMESPACE
@@ -750,7 +750,7 @@ class ComputerUseRemote(Tool):
if isinstance(artifact, dict) and str(artifact.get("encoding", "")).strip().lower() == "base64":
encoded = str(artifact.get("data") or "")
if encoded:
- estimated_size = _estimated_base64_decoded_size(encoded)
+ estimated_size = media_artifacts.estimated_base64_decoded_size(encoded)
if estimated_size > MAX_CAPTURE_ARTIFACT_SIZE_BYTES:
raise RuntimeError(
"Computer-use capture artifact is too large to attach safely "
@@ -759,7 +759,11 @@ class ComputerUseRemote(Tool):
mime = str(artifact.get("mime") or "image/png").strip()
if not mime.startswith("image/"):
mime = "image/png"
- filename = _safe_filename(str(artifact.get("filename") or "computer-use-capture.png"))
+ filename = media_artifacts.safe_filename(
+ str(artifact.get("filename") or "computer-use-capture.png"),
+ default=f"computer-use-{uuid.uuid4().hex}.png",
+ default_extension=".png",
+ )
return f"data:{mime};base64,{encoded}", Path(filename).stem
if path_error is not None:
@@ -845,19 +849,6 @@ class ComputerUseRemote(Tool):
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
-def _safe_filename(value: str) -> str:
- cleaned = "".join(char if char.isalnum() or char in {"-", "_", "."} else "_" for char in value)
- cleaned = cleaned.strip("._") or f"computer-use-{uuid.uuid4().hex}.png"
- if "." not in cleaned:
- cleaned += ".png"
- return cleaned
-
-
-def _estimated_base64_decoded_size(data: str) -> int:
- compact_length = sum(1 for char in data if not char.isspace())
- return (compact_length * 3) // 4
-
-
def _sanitize_tool_text(value: str) -> str:
try:
from helpers.strings import sanitize_string
diff --git a/plugins/_browser/helpers/connector_runtime.py b/plugins/_browser/helpers/connector_runtime.py
index 235a8ea47..1d2a0d4bf 100644
--- a/plugins/_browser/helpers/connector_runtime.py
+++ b/plugins/_browser/helpers/connector_runtime.py
@@ -9,7 +9,7 @@ from pathlib import Path
from typing import Any
from urllib.parse import urlparse
-from helpers import ephemeral_images
+from helpers import ephemeral_images, media_artifacts
try:
from helpers.ws import NAMESPACE
@@ -425,13 +425,17 @@ class ConnectorBrowserRuntime:
data = str(artifact.get("data") or "")
if not data:
return result
- estimated_size = _estimated_base64_decoded_size(data)
+ estimated_size = media_artifacts.estimated_base64_decoded_size(data)
if estimated_size > MAX_ARTIFACT_SIZE_BYTES:
raise RuntimeError(
"Host browser artifact is too large to attach safely "
f"({estimated_size} bytes, limit {MAX_ARTIFACT_SIZE_BYTES} bytes)."
)
- filename = _safe_filename(str(artifact.get("filename") or "host-browser.jpg"))
+ filename = media_artifacts.safe_filename(
+ str(artifact.get("filename") or "host-browser.jpg"),
+ default=f"host-browser-{uuid.uuid4().hex}.jpg",
+ default_extension=".jpg",
+ )
try:
ref = ephemeral_images.put_image(
context_id=self.context_id,
@@ -558,15 +562,3 @@ def _api_base_is_local(api_base: str) -> bool:
hostname = (parsed.hostname or "").strip().lower()
return hostname in _LOCAL_HOSTS
-
-def _safe_filename(value: str) -> str:
- cleaned = "".join(char if char.isalnum() or char in {"-", "_", "."} else "_" for char in value)
- cleaned = cleaned.strip("._") or f"host-browser-{uuid.uuid4().hex}.jpg"
- if "." not in cleaned:
- cleaned += ".jpg"
- return cleaned
-
-
-def _estimated_base64_decoded_size(data: str) -> int:
- compact_length = sum(1 for char in data if not char.isspace())
- return (compact_length * 3) // 4
diff --git a/tests/test_mcp_handler_multimodal.py b/tests/test_mcp_handler_multimodal.py
new file mode 100644
index 000000000..777e23151
--- /dev/null
+++ b/tests/test_mcp_handler_multimodal.py
@@ -0,0 +1,334 @@
+from __future__ import annotations
+
+import asyncio
+import base64
+import importlib
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from types import ModuleType, SimpleNamespace
+
+import pytest
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
+
+
+@dataclass
+class _StubResponse:
+ message: str
+ break_loop: bool
+ additional: dict | None = None
+
+
+class _StubTool:
+ def __init__(
+ self,
+ agent=None,
+ name="",
+ method=None,
+ args=None,
+ message="",
+ loop_data=None,
+ **kwargs,
+ ):
+ self.agent = agent
+ self.name = name
+ self.method = method
+ self.args = args or {}
+ self.message = message
+ self.loop_data = loop_data
+ self.log = None
+
+ def nice_key(self, key: str) -> str:
+ return key
+
+
+class _FakeContent(SimpleNamespace):
+ pass
+
+
+class _FakeCallToolResult(SimpleNamespace):
+ pass
+
+
+def mcp_handler_module(monkeypatch, tmp_path):
+ monkeypatch.delitem(sys.modules, "helpers.mcp_handler", raising=False)
+
+ agent_module = ModuleType("agent")
+ agent_module.AgentContext = type("AgentContext", (), {})
+ agent_module.Agent = type("Agent", (), {})
+ agent_module.LoopData = type("LoopData", (), {})
+ monkeypatch.setitem(sys.modules, "agent", agent_module)
+
+ tool_module = ModuleType("helpers.tool")
+ tool_module.Response = _StubResponse
+ tool_module.Tool = _StubTool
+ monkeypatch.setitem(sys.modules, "helpers.tool", tool_module)
+
+ settings_module = ModuleType("helpers.settings")
+ monkeypatch.setitem(sys.modules, "helpers.settings", settings_module)
+
+ history_module = ModuleType("helpers.history")
+ history_module.RawMessage = lambda **kwargs: dict(kwargs)
+ monkeypatch.setitem(sys.modules, "helpers.history", history_module)
+
+ mcp_module = ModuleType("mcp")
+ mcp_module.ClientSession = type("ClientSession", (), {})
+ mcp_module.StdioServerParameters = type("StdioServerParameters", (), {})
+ monkeypatch.setitem(sys.modules, "mcp", mcp_module)
+
+ mcp_client_stdio = ModuleType("mcp.client.stdio")
+ mcp_client_stdio.stdio_client = lambda *args, **kwargs: None
+ monkeypatch.setitem(sys.modules, "mcp.client.stdio", mcp_client_stdio)
+
+ mcp_client_sse = ModuleType("mcp.client.sse")
+ mcp_client_sse.sse_client = lambda *args, **kwargs: None
+ monkeypatch.setitem(sys.modules, "mcp.client.sse", mcp_client_sse)
+
+ mcp_client_streamable_http = ModuleType("mcp.client.streamable_http")
+ mcp_client_streamable_http.streamablehttp_client = lambda *args, **kwargs: None
+ monkeypatch.setitem(
+ sys.modules,
+ "mcp.client.streamable_http",
+ mcp_client_streamable_http,
+ )
+
+ mcp_shared_message = ModuleType("mcp.shared.message")
+ mcp_shared_message.SessionMessage = type("SessionMessage", (), {})
+ monkeypatch.setitem(sys.modules, "mcp.shared.message", mcp_shared_message)
+
+ mcp_types = ModuleType("mcp.types")
+ mcp_types.CallToolResult = _FakeCallToolResult
+ mcp_types.ListToolsResult = type("ListToolsResult", (), {})
+ monkeypatch.setitem(sys.modules, "mcp.types", mcp_types)
+
+ module = importlib.import_module("helpers.mcp_handler")
+
+ class _SilentPrintStyle:
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def print(self, *args, **kwargs):
+ return self
+
+ def stream(self, *args, **kwargs):
+ return self
+
+ def _fake_get_abs_path(*parts):
+ return str(tmp_path.joinpath(*parts))
+
+ def _fake_normalize_a0_path(path: str) -> str:
+ path_obj = Path(path)
+ try:
+ rel = path_obj.relative_to(tmp_path)
+ except ValueError:
+ return str(path_obj)
+ return "/a0/" + str(rel).replace("\\", "/")
+
+ monkeypatch.setattr(module, "PrintStyle", _SilentPrintStyle)
+ monkeypatch.setattr(module.media_artifacts.files, "get_abs_path", _fake_get_abs_path)
+ monkeypatch.setattr(module.media_artifacts.files, "normalize_a0_path", _fake_normalize_a0_path)
+ return module, tmp_path
+
+
+def _agent_recorder(context_id: str = "ctx-mcp"):
+ tool_results: list[tuple[tuple, dict]] = []
+ messages: list[tuple[tuple, dict]] = []
+ updates: list[dict] = []
+ warnings: list[dict] = []
+ agent = SimpleNamespace(
+ agent_name="Agent Zero",
+ context=SimpleNamespace(
+ id=context_id,
+ log=SimpleNamespace(log=lambda **kwargs: warnings.append(kwargs)),
+ ),
+ hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
+ hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)),
+ )
+ log = SimpleNamespace(id="mcp-log", update=lambda **kwargs: updates.append(kwargs))
+ return agent, log, tool_results, messages, updates, warnings
+
+
+def test_mcp_image_content_becomes_history_image_attachment(mcp_handler_module, monkeypatch):
+ module, _tmp_path = mcp_handler_module
+ agent, log, tool_results, messages, updates, warnings = _agent_recorder()
+ image_b64 = base64.b64encode(b"image-bytes").decode("ascii")
+ result = _FakeCallToolResult(
+ content=[_FakeContent(type="image", data=image_b64, mimeType="image/webp")],
+ isError=False,
+ )
+
+ class _FakeConfig:
+ async def call_tool(self, name, kwargs):
+ return result
+
+ monkeypatch.setattr(module.MCPConfig, "get_instance", lambda: _FakeConfig())
+
+ tool = module.MCPTool(
+ agent=agent,
+ name="venice_image",
+ method=None,
+ args={},
+ message="",
+ loop_data=None,
+ )
+ tool.log = log
+
+ response = asyncio.run(tool.execute())
+
+ assert "[Tool returned no textual content]" not in response.message
+ assert response.message == "MCP returned image attachment (image/webp, 11 bytes)."
+ assert response.additional is not None
+ data_url = response.additional["raw_content"][0]["image_url"]["url"]
+ assert data_url == f"data:image/webp;base64,{image_b64}"
+
+ asyncio.run(tool.after_execution(response))
+
+ assert tool_results[0][0] == ("venice_image", response.message)
+ raw_message = messages[0][1]["content"]
+ assert raw_message["raw_content"][0]["image_url"]["url"] == data_url
+ assert messages[0][1]["tokens"] == module.MCP_MEDIA_TOKENS_ESTIMATE
+ assert updates[-1]["content"] == response.message
+ assert warnings == []
+
+
+def test_mcp_audio_content_is_saved_instead_of_discarded(mcp_handler_module, monkeypatch):
+ module, tmp_path = mcp_handler_module
+ agent, log, tool_results, messages, updates, warnings = _agent_recorder()
+ audio_b64 = base64.b64encode(b"audio-bytes").decode("ascii")
+ result = _FakeCallToolResult(
+ content=[_FakeContent(type="audio", data=audio_b64, mimeType="audio/mpeg")],
+ isError=False,
+ )
+
+ class _FakeConfig:
+ async def call_tool(self, name, kwargs):
+ return result
+
+ monkeypatch.setattr(module.MCPConfig, "get_instance", lambda: _FakeConfig())
+
+ tool = module.MCPTool(
+ agent=agent,
+ name="venice_audio",
+ method=None,
+ args={},
+ message="",
+ loop_data=None,
+ )
+ tool.log = log
+
+ response = asyncio.run(tool.execute())
+
+ assert response.additional is None
+ assert "[Tool returned no textual content]" not in response.message
+ assert "Saved MCP audio attachment (audio/mpeg, 11 bytes) to /a0/tmp/mcp/ctx_mcp/venice_audio/" in response.message
+ saved_path = response.message.split(" to ", 1)[1].rstrip(".")
+ assert (tmp_path / saved_path.removeprefix("/a0/")).exists()
+
+ asyncio.run(tool.after_execution(response))
+
+ assert tool_results[0][0] == ("venice_audio", response.message)
+ assert messages == []
+ assert updates[-1]["content"] == response.message
+ assert warnings == []
+
+
+def test_mcp_image_resource_blob_becomes_history_image_attachment(mcp_handler_module, monkeypatch):
+ module, _tmp_path = mcp_handler_module
+ agent, log, tool_results, messages, updates, warnings = _agent_recorder()
+ image_b64 = base64.b64encode(b"resource-image").decode("ascii")
+ result = _FakeCallToolResult(
+ content=[
+ _FakeContent(
+ type="resource",
+ resource=_FakeContent(
+ uri="memory://venice/image.webp",
+ mimeType="image/webp",
+ blob=image_b64,
+ ),
+ )
+ ],
+ isError=False,
+ )
+
+ class _FakeConfig:
+ async def call_tool(self, name, kwargs):
+ return result
+
+ monkeypatch.setattr(module.MCPConfig, "get_instance", lambda: _FakeConfig())
+
+ tool = module.MCPTool(
+ agent=agent,
+ name="venice_resource_image",
+ method=None,
+ args={},
+ message="",
+ loop_data=None,
+ )
+ tool.log = log
+
+ response = asyncio.run(tool.execute())
+
+ assert response.message == "MCP returned resource image attachment (image/webp, 14 bytes)."
+ assert response.additional is not None
+ data_url = response.additional["raw_content"][0]["image_url"]["url"]
+ assert data_url == f"data:image/webp;base64,{image_b64}"
+
+ asyncio.run(tool.after_execution(response))
+
+ assert tool_results[0][0] == ("venice_resource_image", response.message)
+ raw_message = messages[0][1]["content"]
+ assert raw_message["raw_content"][0]["image_url"]["url"] == data_url
+ assert updates[-1]["content"] == response.message
+ assert warnings == []
+
+
+def test_mcp_resource_text_is_preserved(mcp_handler_module, monkeypatch):
+ module, _tmp_path = mcp_handler_module
+ agent, log, tool_results, messages, updates, warnings = _agent_recorder()
+ result = _FakeCallToolResult(
+ content=[
+ _FakeContent(
+ type="resource",
+ resource=_FakeContent(
+ uri="memory://venice/caption.txt",
+ mimeType="text/plain",
+ text="Generated caption text",
+ ),
+ )
+ ],
+ isError=False,
+ )
+
+ class _FakeConfig:
+ async def call_tool(self, name, kwargs):
+ return result
+
+ monkeypatch.setattr(module.MCPConfig, "get_instance", lambda: _FakeConfig())
+
+ tool = module.MCPTool(
+ agent=agent,
+ name="venice_resource",
+ method=None,
+ args={},
+ message="",
+ loop_data=None,
+ )
+ tool.log = log
+
+ response = asyncio.run(tool.execute())
+
+ assert response.additional is None
+ assert "Resource memory://venice/caption.txt:" in response.message
+ assert "Generated caption text" in response.message
+
+ asyncio.run(tool.after_execution(response))
+
+ assert tool_results[0][0] == ("venice_resource", response.message)
+ assert messages == []
+ assert updates[-1]["content"] == response.message
+ assert warnings == []
diff --git a/tests/test_media_artifacts.py b/tests/test_media_artifacts.py
new file mode 100644
index 000000000..1bbc328f9
--- /dev/null
+++ b/tests/test_media_artifacts.py
@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import base64
+import sys
+from pathlib import Path
+
+import pytest
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
+
+from helpers import media_artifacts
+
+
+def test_image_data_url_from_base64_compacts_and_normalizes_mime():
+ encoded = " \n" + base64.b64encode(b"image-bytes").decode("ascii") + "\n"
+
+ image = media_artifacts.image_data_url_from_base64(
+ encoded,
+ mime_type="IMAGE/WEBP",
+ )
+
+ assert image.mime == "image/webp"
+ assert image.size == 11
+ assert image.url == "data:image/webp;base64,aW1hZ2UtYnl0ZXM="
+
+
+def test_decode_base64_payload_rejects_empty_invalid_and_oversized_data():
+ with pytest.raises(media_artifacts.EmptyBase64Data):
+ media_artifacts.decode_base64_payload(" \n ")
+
+ with pytest.raises(media_artifacts.InvalidBase64Data):
+ media_artifacts.decode_base64_payload("not base64")
+
+ with pytest.raises(media_artifacts.ArtifactTooLarge) as exc_info:
+ media_artifacts.decode_base64_payload("ZmFrZQ==", max_bytes=2)
+
+ assert exc_info.value.size == media_artifacts.estimated_base64_decoded_size("ZmFrZQ==")
+ assert exc_info.value.limit == 2
+
+
+def test_save_base64_artifact_uses_uri_filename_and_normalized_a0_path(monkeypatch, tmp_path):
+ def fake_get_abs_path(*parts):
+ return str(tmp_path.joinpath(*parts))
+
+ def fake_normalize_a0_path(path: str):
+ return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/")
+
+ monkeypatch.setattr(media_artifacts.files, "get_abs_path", fake_get_abs_path)
+ monkeypatch.setattr(media_artifacts.files, "normalize_a0_path", fake_normalize_a0_path)
+
+ artifact = media_artifacts.save_base64_artifact(
+ base64.b64encode(b"audio-bytes").decode("ascii"),
+ mime_type="audio/mpeg",
+ directory_parts=("tmp", "media-test"),
+ preferred_name="memory://venice/generated track.mp3",
+ default_filename="fallback.bin",
+ )
+
+ assert artifact.mime == "audio/mpeg"
+ assert artifact.size == 11
+ assert artifact.path.startswith("/a0/tmp/media-test/generated_track_")
+ assert artifact.path.endswith(".mp3")
+ assert (tmp_path / artifact.path.removeprefix("/a0/")).read_bytes() == b"audio-bytes"