Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Enforce Anthropic’s 2000px limit for requests with many images

Flightdeck Agent0b540f3eb61c
diff --git a/backend/agent/providers/anthropic/image.py b/backend/agent/providers/anthropic/image.py
index e3ba92dc..4e77de6a 100644
--- a/backend/agent/providers/anthropic/image.py
+++ b/backend/agent/providers/anthropic/image.py
@@ -9,14 +9,12 @@ Comparison with official Anthropic docs
Aligned:
- 5 MB per-image size limit matches the documented API maximum.
+ - Requests with more than 20 images use the stricter 2000x2000 px limit.
- Output uses the correct base64 source format (type, media_type, data).
Divergences:
- - Max dimension is set to 7990 px as a safety margin; the API rejects at
- 8000 px. This is intentionally conservative.
- - The docs note that when >20 images are sent in a single request the
- per-image limit drops to 2000x2000 px. We do not enforce that stricter
- limit here (the app typically sends far fewer images).
+ - The normal max dimension is set to 7990 px as a safety margin; the API
+ rejects at 8000 px. This is intentionally conservative.
- JPEG conversion drops alpha channels, which is acceptable for website
screenshots but would degrade transparent PNGs.
@@ -42,8 +40,17 @@ CLAUDE_IMAGE_MAX_SIZE = 5 * 1024 * 1024
# docstring).
CLAUDE_MAX_IMAGE_DIMENSION = 7990
+# Once a request contains more than 20 images, Anthropic lowers the per-image
+# dimension limit to 2000x2000 px.
+CLAUDE_MANY_IMAGE_THRESHOLD = 20
+CLAUDE_MANY_IMAGE_MAX_DIMENSION = 2000
-def process_image(image_data_url: str) -> tuple[str, str]:
+
+def process_image(
+ image_data_url: str,
+ *,
+ max_dimension: int = CLAUDE_MAX_IMAGE_DIMENSION,
+) -> tuple[str, str]:
"""Resize / compress a data-URL image to fit Claude's vision limits.
Returns (media_type, base64_data) suitable for an ``image`` content block.
@@ -52,17 +59,27 @@ def process_image(image_data_url: str) -> tuple[str, str]:
base64_data = image_data_url.split(",")[1]
image_bytes = base64.b64decode(base64_data)
- return process_image_bytes(image_bytes, media_type)
+ return process_image_bytes(
+ image_bytes,
+ media_type,
+ max_dimension=max_dimension,
+ )
-def process_image_bytes(image_bytes: bytes, media_type: str) -> tuple[str, str]:
+def process_image_bytes(
+ image_bytes: bytes,
+ media_type: str,
+ *,
+ max_dimension: int = CLAUDE_MAX_IMAGE_DIMENSION,
+) -> tuple[str, str]:
"""Resize / compress image bytes to fit Claude's vision limits."""
+ if max_dimension <= 0:
+ raise ValueError("max_dimension must be positive")
img = Image.open(io.BytesIO(image_bytes))
base64_data = base64.b64encode(image_bytes).decode("utf-8")
is_under_dimension_limit = (
- img.width < CLAUDE_MAX_IMAGE_DIMENSION
- and img.height < CLAUDE_MAX_IMAGE_DIMENSION
+ img.width <= max_dimension and img.height <= max_dimension
)
is_under_size_limit = len(base64_data) <= CLAUDE_IMAGE_MAX_SIZE
@@ -73,11 +90,11 @@ def process_image_bytes(image_bytes: bytes, media_type: str) -> tuple[str, str]:
if not is_under_dimension_limit:
if img.width > img.height:
- new_width = CLAUDE_MAX_IMAGE_DIMENSION
- new_height = int((CLAUDE_MAX_IMAGE_DIMENSION / img.width) * img.height)
+ new_width = max_dimension
+ new_height = max(1, int((max_dimension / img.width) * img.height))
else:
- new_height = CLAUDE_MAX_IMAGE_DIMENSION
- new_width = int((CLAUDE_MAX_IMAGE_DIMENSION / img.height) * img.width)
+ new_height = max_dimension
+ new_width = max(1, int((max_dimension / img.height) * img.width))
img = img.resize((new_width, new_height), Image.DEFAULT_STRATEGY)
diff --git a/backend/agent/providers/anthropic/provider.py b/backend/agent/providers/anthropic/provider.py
index b0d66ec8..968e956d 100644
--- a/backend/agent/providers/anthropic/provider.py
+++ b/backend/agent/providers/anthropic/provider.py
@@ -1,4 +1,5 @@
# pyright: reportUnknownVariableType=false
+import base64
import copy
import json
import uuid
@@ -15,7 +16,12 @@ from agent.providers.base import (
ProviderTurn,
StreamEvent,
)
-from agent.providers.anthropic.image import process_image, process_image_bytes
+from agent.providers.anthropic.image import (
+ CLAUDE_MANY_IMAGE_MAX_DIMENSION,
+ CLAUDE_MANY_IMAGE_THRESHOLD,
+ process_image,
+ process_image_bytes,
+)
from agent.providers.pricing import MODEL_PRICING
from agent.providers.token_usage import TokenUsage
from agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments
@@ -64,6 +70,63 @@ def _get_anthropic_effort(model: Llm) -> str:
return "max"
+def _anthropic_image_blocks(
+ messages: List[Dict[str, Any]],
+) -> List[Dict[str, Any]]:
+ """Return direct and tool-result image blocks from Anthropic messages."""
+ image_blocks: List[Dict[str, Any]] = []
+
+ def collect(content: Any) -> None:
+ if not isinstance(content, list):
+ return
+ for block in content:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") == "image":
+ image_blocks.append(block)
+ elif block.get("type") == "tool_result":
+ collect(block.get("content"))
+
+ for message in messages:
+ collect(message.get("content"))
+
+ return image_blocks
+
+
+def _enforce_many_image_dimension_limit(
+ messages: List[Dict[str, Any]],
+) -> bool:
+ """Resize base64 images when a request crosses Anthropic's 20-image limit.
+
+ Returns whether the request is subject to the many-image limit. URL-backed
+ images count toward the threshold, even though only local base64 sources can
+ be resized here.
+ """
+ image_blocks = _anthropic_image_blocks(messages)
+ if len(image_blocks) <= CLAUDE_MANY_IMAGE_THRESHOLD:
+ return False
+
+ for block in image_blocks:
+ source = block.get("source")
+ if not isinstance(source, dict) or source.get("type") != "base64":
+ continue
+
+ media_type = source.get("media_type")
+ encoded_data = source.get("data")
+ if not isinstance(media_type, str) or not isinstance(encoded_data, str):
+ continue
+
+ processed_media_type, processed_data = process_image_bytes(
+ base64.b64decode(encoded_data),
+ media_type,
+ max_dimension=CLAUDE_MANY_IMAGE_MAX_DIMENSION,
+ )
+ source["media_type"] = processed_media_type
+ source["data"] = processed_data
+
+ return True
+
+
def _convert_openai_messages_to_claude(
messages: List[ChatCompletionMessageParam],
) -> tuple[str, List[Dict[str, Any]]]:
@@ -71,6 +134,18 @@ def _convert_openai_messages_to_claude(
system_prompt = cast(str, cloned_messages[0].get("content"))
claude_messages = [dict(message) for message in cloned_messages[1:]]
+ image_count = sum(
+ 1
+ for message in claude_messages
+ if isinstance(message.get("content"), list)
+ for content in cast(List[Dict[str, Any]], message["content"])
+ if content.get("type") == "image_url"
+ )
+ max_dimension = (
+ CLAUDE_MANY_IMAGE_MAX_DIMENSION
+ if image_count > CLAUDE_MANY_IMAGE_THRESHOLD
+ else None
+ )
for message in claude_messages:
if not isinstance(message["content"], list):
@@ -82,7 +157,13 @@ def _convert_openai_messages_to_claude(
content["type"] = "image"
image_data_url = cast(str, content["image_url"]["url"])
- media_type, base64_data = process_image(image_data_url)
+ if max_dimension is None:
+ media_type, base64_data = process_image(image_data_url)
+ else:
+ media_type, base64_data = process_image(
+ image_data_url,
+ max_dimension=max_dimension,
+ )
del content["image_url"]
content["source"] = {
"type": "base64",
@@ -246,8 +327,22 @@ class AnthropicProviderSession(ProviderSession):
system_prompt, claude_messages = _convert_openai_messages_to_claude(prompt_messages)
self._system_prompt = system_prompt
self._messages = claude_messages
+ self._many_image_limit_active = (
+ len(_anthropic_image_blocks(self._messages))
+ > CLAUDE_MANY_IMAGE_THRESHOLD
+ )
+
+ def _ensure_many_image_dimension_limit(self) -> None:
+ if self._many_image_limit_active:
+ return
+ self._many_image_limit_active = _enforce_many_image_dimension_limit(
+ self._messages
+ )
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
+ # Tool screenshots accumulate across turns. Re-check before every API
+ # call so crossing 20 images cannot leave earlier images above 2000 px.
+ self._ensure_many_image_dimension_limit()
stream_kwargs: Dict[str, Any] = {
"model": _get_anthropic_api_model_name(self._model),
"max_tokens": 50000,
@@ -291,8 +386,7 @@ class AnthropicProviderSession(ProviderSession):
assistant_turn=final_message,
)
- @staticmethod
- def _image_block(part: Any) -> Dict[str, Any] | None:
+ def _image_block(self, part: Any) -> Dict[str, Any] | None:
"""A public URL goes as a url source; local bytes go as base64."""
if part.image_url:
return {
@@ -300,7 +394,17 @@ class AnthropicProviderSession(ProviderSession):
"source": {"type": "url", "url": part.image_url},
}
if part.data is not None:
- media_type, base64_data = process_image_bytes(part.data, part.mime_type)
+ if self._many_image_limit_active:
+ media_type, base64_data = process_image_bytes(
+ part.data,
+ part.mime_type,
+ max_dimension=CLAUDE_MANY_IMAGE_MAX_DIMENSION,
+ )
+ else:
+ media_type, base64_data = process_image_bytes(
+ part.data,
+ part.mime_type,
+ )
return {
"type": "image",
"source": {
@@ -360,6 +464,7 @@ class AnthropicProviderSession(ProviderSession):
)
self._messages.append({"role": "user", "content": tool_result_blocks})
+ self._ensure_many_image_dimension_limit()
async def close(self) -> None:
u = self._total_usage
diff --git a/backend/tests/test_anthropic_many_image_limit.py b/backend/tests/test_anthropic_many_image_limit.py
new file mode 100644
index 00000000..30c2e3ac
--- /dev/null
+++ b/backend/tests/test_anthropic_many_image_limit.py
@@ -0,0 +1,139 @@
+import base64
+import io
+from typing import Any, List, cast
+
+import pytest
+from openai.types.chat import ChatCompletionMessageParam
+from PIL import Image
+
+from agent.providers.anthropic.image import CLAUDE_MANY_IMAGE_MAX_DIMENSION
+from agent.providers.anthropic.provider import AnthropicProviderSession
+from agent.providers.base import ExecutedToolCall, ProviderTurn
+from agent.tools.types import ToolCall, ToolExecutionResult, ToolMultimodalPart
+from llm import Llm
+
+
+def _png_bytes(width: int, height: int) -> bytes:
+ output = io.BytesIO()
+ Image.new("RGB", (width, height), "white").save(output, format="PNG")
+ return output.getvalue()
+
+
+def _data_url(image_bytes: bytes) -> str:
+ encoded = base64.b64encode(image_bytes).decode("ascii")
+ return f"data:image/png;base64,{encoded}"
+
+
+def _prompt_messages(image_count: int) -> List[ChatCompletionMessageParam]:
+ oversized = _data_url(_png_bytes(2100, 100))
+ small = _data_url(_png_bytes(1, 1))
+ content: list[dict[str, Any]] = [
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": oversized if index == 0 else small,
+ "detail": "high",
+ },
+ }
+ for index in range(image_count)
+ ]
+ content.append({"type": "text", "text": "Build this page"})
+ return cast(
+ List[ChatCompletionMessageParam],
+ [
+ {"role": "system", "content": "system"},
+ {"role": "user", "content": content},
+ ],
+ )
+
+
+def _image_dimensions(source: dict[str, Any]) -> tuple[int, int]:
+ image_bytes = base64.b64decode(source["data"])
+ with Image.open(io.BytesIO(image_bytes)) as image:
+ return image.size
+
+
+def _session(image_count: int) -> AnthropicProviderSession:
+ return AnthropicProviderSession(
+ client=object(), # type: ignore[arg-type]
+ model=Llm.CLAUDE_FABLE_5_HIGH,
+ prompt_messages=_prompt_messages(image_count),
+ tools=[],
+ )
+
+
+def _first_prompt_image_source(session: AnthropicProviderSession) -> dict[str, Any]:
+ content = session._messages[0]["content"]
+ assert isinstance(content, list)
+ first_image = cast(dict[str, Any], content[0])
+ source = first_image["source"]
+ assert isinstance(source, dict)
+ return cast(dict[str, Any], source)
+
+
+def _executed_image(image_bytes: bytes, call_id: str) -> ExecutedToolCall:
+ return ExecutedToolCall(
+ tool_call=ToolCall(id=call_id, name="screenshot_preview", arguments={}),
+ result=ToolExecutionResult(
+ ok=True,
+ result={"status": "ok"},
+ summary={},
+ multimodal_parts=[
+ ToolMultimodalPart(
+ display_name="preview.png",
+ mime_type="image/png",
+ data=image_bytes,
+ )
+ ],
+ ),
+ )
+
+
+ ("image_count", "expected_width"),
+ [
+ (20, 2100),
+ (21, CLAUDE_MANY_IMAGE_MAX_DIMENSION),
+ ],
+)
+def test_initial_request_uses_stricter_limit_only_above_twenty_images(
+ image_count: int,
+ expected_width: int,
+) -> None:
+ session = _session(image_count)
+
+ assert _image_dimensions(_first_prompt_image_source(session))[0] == expected_width
+
+
+async def test_crossing_limit_with_tool_images_resizes_history_and_future_images() -> None:
+ session = _session(20)
+ assert _image_dimensions(_first_prompt_image_source(session))[0] == 2100
+
+ await session.append_tool_results(
+ ProviderTurn(assistant_text="", tool_calls=[], assistant_turn=None),
+ [_executed_image(_png_bytes(1, 1), "call-1")],
+ )
+
+ assert session._many_image_limit_active is True
+ assert (
+ _image_dimensions(_first_prompt_image_source(session))[0]
+ == CLAUDE_MANY_IMAGE_MAX_DIMENSION
+ )
+
+ await session.append_tool_results(
+ ProviderTurn(assistant_text="", tool_calls=[], assistant_turn=None),
+ [_executed_image(_png_bytes(2200, 100), "call-2")],
+ )
+ latest_tool_results = session._messages[-1]["content"]
+ assert isinstance(latest_tool_results, list)
+ tool_result = cast(dict[str, Any], latest_tool_results[0])
+ latest_content = tool_result["content"]
+ assert isinstance(latest_content, list)
+ image_blocks = cast(list[dict[str, Any]], latest_content)
+ latest_image = next(block for block in image_blocks if block["type"] == "image")
+
+ assert (
+ _image_dimensions(latest_image["source"])[0]
+ == CLAUDE_MANY_IMAGE_MAX_DIMENSION
+ )