Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Enforce ToolMultimodalPart source invariant

Abi Rajacaac34d05acc
diff --git a/backend/agent/tools/local_assets.py b/backend/agent/tools/local_assets.py
index f412d01f..0c1000aa 100644
--- a/backend/agent/tools/local_assets.py
+++ b/backend/agent/tools/local_assets.py
@@ -16,6 +16,15 @@ from config import LOCAL_ASSET_DIR
LOCAL_ASSET_HOSTS = {"127.0.0.1", "localhost", "::1"}
+def is_local_host_url(url: str) -> bool:
+ """True if the URL points at this host (a loopback address).
+
+ Such URLs are reachable from this backend but NOT from cloud model APIs
+ (Anthropic, OpenAI), so they must never be handed to a model as a URL.
+ """
+ return (urlparse(url).hostname or "").lower() in LOCAL_ASSET_HOSTS
+
+
def guess_image_mime(url: str) -> str:
"""Best-effort image MIME from a URL's extension; defaults to PNG."""
return mimetypes.guess_type(urlparse(url).path)[0] or "image/png"
diff --git a/backend/agent/tools/types.py b/backend/agent/tools/types.py
index 706b1d7b..295bdc02 100644
--- a/backend/agent/tools/types.py
+++ b/backend/agent/tools/types.py
@@ -1,6 +1,8 @@
from dataclasses import dataclass
from typing import Any, Dict, List, Optional
+from agent.tools.local_assets import is_local_host_url
+
@dataclass(frozen=True)
class ToolCall:
@@ -24,6 +26,21 @@ class ToolMultimodalPart:
data: Optional[bytes] = None
image_url: Optional[str] = None
+ def __post_init__(self) -> None:
+ # Enforce the source invariant instead of trusting callers: exactly one
+ # of data/image_url, and image_url must be model-reachable. A localhost
+ # URL here would silently break Anthropic/OpenAI (they can't fetch it)
+ # while only Gemini's download fallback worked.
+ if (self.data is None) == (self.image_url is None):
+ raise ValueError(
+ "ToolMultimodalPart requires exactly one of `data` or `image_url`."
+ )
+ if self.image_url is not None and is_local_host_url(self.image_url):
+ raise ValueError(
+ "ToolMultimodalPart.image_url must be publicly fetchable, got a "
+ f"localhost URL: {self.image_url!r}. Pass local images as `data` bytes."
+ )
+
@dataclass
class ToolExecutionResult:
diff --git a/backend/tests/test_tool_image_outputs.py b/backend/tests/test_tool_image_outputs.py
index 47b4a1e5..514c905f 100644
--- a/backend/tests/test_tool_image_outputs.py
+++ b/backend/tests/test_tool_image_outputs.py
@@ -27,6 +27,29 @@ LOCAL = ToolMultimodalPart(
)
+def test_multimodal_part_enforces_source_invariant() -> None:
+ # Valid: exactly one source.
+ ToolMultimodalPart(display_name="a", mime_type="image/png", data=b"x")
+ ToolMultimodalPart(
+ display_name="a", mime_type="image/png", image_url="https://x/y.png"
+ )
+ # Neither source.
+ with pytest.raises(ValueError):
+ ToolMultimodalPart(display_name="a", mime_type="image/png")
+ # Both sources.
+ with pytest.raises(ValueError):
+ ToolMultimodalPart(
+ display_name="a", mime_type="image/png", data=b"x", image_url="https://x/y"
+ )
+ # A localhost URL isn't model-reachable and must be rejected.
+ with pytest.raises(ValueError, match="publicly fetchable"):
+ ToolMultimodalPart(
+ display_name="a",
+ mime_type="image/png",
+ image_url="http://127.0.0.1:7011/local-assets/z.png",
+ )
+
+
def _executed(parts: List[ToolMultimodalPart], ok: bool = True) -> ExecutedToolCall:
return ExecutedToolCall(
tool_call=ToolCall(id="call-1", name="extract_assets", arguments={}),