Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Process Anthropic tool images before sending
Abi Raja244fa1783290
diff --git a/backend/agent/providers/anthropic/image.py b/backend/agent/providers/anthropic/image.pyindex 77c889a3..e3ba92dc 100644--- a/backend/agent/providers/anthropic/image.py+++ b/backend/agent/providers/anthropic/image.py@@ -52,7 +52,13 @@ def process_image(image_data_url: str) -> tuple[str, str]:base64_data = image_data_url.split(",")[1]image_bytes = base64.b64decode(base64_data)+ return process_image_bytes(image_bytes, media_type)+++def process_image_bytes(image_bytes: bytes, media_type: str) -> tuple[str, str]:+ """Resize / compress image bytes to fit Claude's vision limits."""img = Image.open(io.BytesIO(image_bytes))+ base64_data = base64.b64encode(image_bytes).decode("utf-8")is_under_dimension_limit = (img.width < CLAUDE_MAX_IMAGE_DIMENSIONdiff --git a/backend/agent/providers/anthropic/provider.py b/backend/agent/providers/anthropic/provider.pyindex cdf0fd48..b0d66ec8 100644--- a/backend/agent/providers/anthropic/provider.py+++ b/backend/agent/providers/anthropic/provider.py@@ -1,5 +1,4 @@# pyright: reportUnknownVariableType=false-import base64import copyimport jsonimport uuid@@ -16,7 +15,7 @@ from agent.providers.base import (ProviderTurn,StreamEvent,)-from agent.providers.anthropic.image import process_image+from agent.providers.anthropic.image import process_image, process_image_bytesfrom agent.providers.pricing import MODEL_PRICINGfrom agent.providers.token_usage import TokenUsagefrom agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments@@ -301,12 +300,13 @@ class AnthropicProviderSession(ProviderSession):"source": {"type": "url", "url": part.image_url},}if part.data is not None:+ media_type, base64_data = process_image_bytes(part.data, part.mime_type)return {"type": "image","source": {"type": "base64",- "media_type": part.mime_type,- "data": base64.b64encode(part.data).decode("ascii"),+ "media_type": media_type,+ "data": base64_data,},}return Nonediff --git a/backend/tests/test_tool_image_outputs.py b/backend/tests/test_tool_image_outputs.pyindex bcc13e1e..12581f20 100644--- a/backend/tests/test_tool_image_outputs.py+++ b/backend/tests/test_tool_image_outputs.py@@ -22,8 +22,11 @@ PUBLIC = ToolMultimodalPart(mime_type="image/png",image_url="https://replicate.delivery/abc/out.png",)+LOCAL_IMAGE_BYTES = base64.b64decode(+ "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="+)LOCAL = ToolMultimodalPart(- display_name="crop.png", mime_type="image/png", data=b"local-bytes"+ display_name="crop.png", mime_type="image/png", data=LOCAL_IMAGE_BYTES)@@ -60,7 +63,20 @@ def _executed(parts: List[ToolMultimodalPart], ok: bool = True) -> ExecutedToolC@pytest.mark.asyncio-async def test_anthropic_url_for_public_base64_for_local() -> None:+async def test_anthropic_url_for_public_processes_local_bytes(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ processed_data = "processed-image"+ captured_images: list[tuple[bytes, str]] = []++ def fake_process_image_bytes(image_bytes: bytes, media_type: str) -> tuple[str, str]:+ captured_images.append((image_bytes, media_type))+ return ("image/jpeg", processed_data)++ monkeypatch.setattr(+ "agent.providers.anthropic.provider.process_image_bytes",+ fake_process_image_bytes,+ )session = AnthropicProviderSession(client=object(), # type: ignore[arg-type]model=Llm.CLAUDE_OPUS_4_8_HIGH,@@ -77,8 +93,12 @@ async def test_anthropic_url_for_public_base64_for_local() -> None:content = session._messages[-1]["content"][0]["content"] # type: ignore[index]images = [b for b in content if b.get("type") == "image"]assert images[0]["source"] == {"type": "url", "url": PUBLIC.image_url}- assert images[1]["source"]["type"] == "base64"- assert images[1]["source"]["data"] == base64.b64encode(b"local-bytes").decode()+ assert captured_images == [(LOCAL_IMAGE_BYTES, "image/png")]+ assert images[1]["source"] == {+ "type": "base64",+ "media_type": "image/jpeg",+ "data": processed_data,+ }@pytest.mark.asyncio@@ -119,7 +139,7 @@ async def test_openai_url_for_public_dataurl_for_local_with_detail() -> None:images = [p for p in output if p.get("type") == "input_image"]assert images[0]["image_url"] == PUBLIC.image_urlassert images[1]["image_url"] == (- "data:image/png;base64," + base64.b64encode(b"local-bytes").decode()+ "data:image/png;base64," + base64.b64encode(LOCAL_IMAGE_BYTES).decode())assert all("detail" in p for p in images)@@ -174,7 +194,7 @@ async def test_gemini_inlines_local_bytes_and_fetches_public_url(assert response_parts[0].inline_data.data == b"fetched-bytes"assert response_parts[0].inline_data.mime_type == "image/webp"# Local bytes inlined as-is.- assert response_parts[1].inline_data.data == b"local-bytes"+ assert response_parts[1].inline_data.data == LOCAL_IMAGE_BYTES@pytest.mark.asyncio