Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Process Anthropic tool images before sending

Abi Raja244fa1783290
diff --git a/backend/agent/providers/anthropic/image.py b/backend/agent/providers/anthropic/image.py
index 77c889a3..e3ba92dc 100644
--- a/backend/agent/providers/anthropic/image.py
+++ b/backend/agent/providers/anthropic/image.py
@@ -52,7 +52,13 @@ def process_image(image_data_url: str) -> tuple[str, str]:
base64_data = image_data_url.split(",")[1]
image_bytes = base64.b64decode(base64_data)
+ return process_image_bytes(image_bytes, media_type)
+
+
+def process_image_bytes(image_bytes: bytes, media_type: str) -> tuple[str, str]:
+ """Resize / compress image bytes to fit Claude's vision limits."""
img = Image.open(io.BytesIO(image_bytes))
+ base64_data = base64.b64encode(image_bytes).decode("utf-8")
is_under_dimension_limit = (
img.width < CLAUDE_MAX_IMAGE_DIMENSION
diff --git a/backend/agent/providers/anthropic/provider.py b/backend/agent/providers/anthropic/provider.py
index cdf0fd48..b0d66ec8 100644
--- a/backend/agent/providers/anthropic/provider.py
+++ b/backend/agent/providers/anthropic/provider.py
@@ -1,5 +1,4 @@
# pyright: reportUnknownVariableType=false
-import base64
import copy
import json
import uuid
@@ -16,7 +15,7 @@ from agent.providers.base import (
ProviderTurn,
StreamEvent,
)
-from agent.providers.anthropic.image import process_image
+from agent.providers.anthropic.image import process_image, process_image_bytes
from agent.providers.pricing import MODEL_PRICING
from agent.providers.token_usage import TokenUsage
from agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments
@@ -301,12 +300,13 @@ class AnthropicProviderSession(ProviderSession):
"source": {"type": "url", "url": part.image_url},
}
if part.data is not None:
+ media_type, base64_data = process_image_bytes(part.data, part.mime_type)
return {
"type": "image",
"source": {
"type": "base64",
- "media_type": part.mime_type,
- "data": base64.b64encode(part.data).decode("ascii"),
+ "media_type": media_type,
+ "data": base64_data,
},
}
return None
diff --git a/backend/tests/test_tool_image_outputs.py b/backend/tests/test_tool_image_outputs.py
index bcc13e1e..12581f20 100644
--- a/backend/tests/test_tool_image_outputs.py
+++ b/backend/tests/test_tool_image_outputs.py
@@ -22,8 +22,11 @@ PUBLIC = ToolMultimodalPart(
mime_type="image/png",
image_url="https://replicate.delivery/abc/out.png",
)
+LOCAL_IMAGE_BYTES = base64.b64decode(
+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="
+)
LOCAL = ToolMultimodalPart(
- display_name="crop.png", mime_type="image/png", data=b"local-bytes"
+ display_name="crop.png", mime_type="image/png", data=LOCAL_IMAGE_BYTES
)
@@ -60,7 +63,20 @@ def _executed(parts: List[ToolMultimodalPart], ok: bool = True) -> ExecutedToolC
@pytest.mark.asyncio
-async def test_anthropic_url_for_public_base64_for_local() -> None:
+async def test_anthropic_url_for_public_processes_local_bytes(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ processed_data = "processed-image"
+ captured_images: list[tuple[bytes, str]] = []
+
+ def fake_process_image_bytes(image_bytes: bytes, media_type: str) -> tuple[str, str]:
+ captured_images.append((image_bytes, media_type))
+ return ("image/jpeg", processed_data)
+
+ monkeypatch.setattr(
+ "agent.providers.anthropic.provider.process_image_bytes",
+ fake_process_image_bytes,
+ )
session = AnthropicProviderSession(
client=object(), # type: ignore[arg-type]
model=Llm.CLAUDE_OPUS_4_8_HIGH,
@@ -77,8 +93,12 @@ async def test_anthropic_url_for_public_base64_for_local() -> None:
content = session._messages[-1]["content"][0]["content"] # type: ignore[index]
images = [b for b in content if b.get("type") == "image"]
assert images[0]["source"] == {"type": "url", "url": PUBLIC.image_url}
- assert images[1]["source"]["type"] == "base64"
- assert images[1]["source"]["data"] == base64.b64encode(b"local-bytes").decode()
+ assert captured_images == [(LOCAL_IMAGE_BYTES, "image/png")]
+ assert images[1]["source"] == {
+ "type": "base64",
+ "media_type": "image/jpeg",
+ "data": processed_data,
+ }
@pytest.mark.asyncio
@@ -119,7 +139,7 @@ async def test_openai_url_for_public_dataurl_for_local_with_detail() -> None:
images = [p for p in output if p.get("type") == "input_image"]
assert images[0]["image_url"] == PUBLIC.image_url
assert images[1]["image_url"] == (
- "data:image/png;base64," + base64.b64encode(b"local-bytes").decode()
+ "data:image/png;base64," + base64.b64encode(LOCAL_IMAGE_BYTES).decode()
)
assert all("detail" in p for p in images)
@@ -174,7 +194,7 @@ async def test_gemini_inlines_local_bytes_and_fetches_public_url(
assert response_parts[0].inline_data.data == b"fetched-bytes"
assert response_parts[0].inline_data.mime_type == "image/webp"
# Local bytes inlined as-is.
- assert response_parts[1].inline_data.data == b"local-bytes"
+ assert response_parts[1].inline_data.data == LOCAL_IMAGE_BYTES
@pytest.mark.asyncio