Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Keep images returned by tools (#10762)

Nilayffb5539d2f07
diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py
index eb44107c1fc..d7e83d8a039 100644
--- a/studio/backend/core/inference/anthropic_compat.py
+++ b/studio/backend/core/inference/anthropic_compat.py
@@ -40,6 +40,9 @@ def anthropic_tool_use_id(upstream_id = None) -> str:
return f"toolu_{uuid.uuid4().hex[:24]}"
+TOOL_RESULT_IMAGE_OMITTED = "[image omitted: this model cannot view images]"
+
+
def _anthropic_image_block_to_openai_part(block: dict) -> Optional[dict]:
"""Translate one Anthropic ``image`` block to an OpenAI ``image_url`` part.
@@ -49,11 +52,13 @@ def _anthropic_image_block_to_openai_part(block: dict) -> Optional[dict]:
Returns ``None`` when the source is malformed so the caller can skip it.
"""
- source = block.get("source") or {}
+ source = block.get("source")
+ if not isinstance(source, dict):
+ return None
stype = source.get("type")
if stype == "base64":
data = source.get("data")
- if not data:
+ if not isinstance(data, str) or not data:
return None
media_type = source.get("media_type") or "image/jpeg"
return {
@@ -62,7 +67,7 @@ def _anthropic_image_block_to_openai_part(block: dict) -> Optional[dict]:
}
if stype == "url":
url = source.get("url")
- if not url:
+ if not isinstance(url, str) or not url:
return None
return {"type": "image_url", "image_url": {"url": url}}
return None
@@ -72,6 +77,7 @@ def anthropic_messages_to_openai(
messages: list[dict],
system: Optional[Union[str, list]] = None,
preserve_thinking: bool = False,
+ tool_result_images: bool = True,
) -> list[dict]:
"""Convert Anthropic messages + system to OpenAI-format message dicts.
@@ -79,6 +85,9 @@ def anthropic_messages_to_openai(
content arrays (``[{type: "text", ...}, {type: "image_url", ...}]``) so
they flow through llama-server's native vision pathway.
+ ``tool_result_images=False`` turns tool-result images into a text note for a
+ text-only model; clients resend history, so rejecting would fail every later turn.
+
``preserve_thinking`` keeps replayed assistant ``thinking`` blocks as
``reasoning_content`` on the converted message, so templates that render
historical reasoning (Qwen3.6-style ``preserve_thinking``) actually receive
@@ -162,14 +171,31 @@ def anthropic_messages_to_openai(
elif btype == "tool_result":
tc = b.get("content", "")
if isinstance(tc, list):
- tc = " ".join(
- p["text"] for p in tc if isinstance(p, dict) and p.get("type") == "text"
+ parts = []
+ for item in tc:
+ if not isinstance(item, dict):
+ continue
+ if item.get("type") == "text":
+ parts.append({"type": "text", "text": item["text"]})
+ elif item.get("type") == "image":
+ if not tool_result_images:
+ parts.append(
+ {"type": "text", "text": TOOL_RESULT_IMAGE_OMITTED}
+ )
+ continue
+ part = _anthropic_image_block_to_openai_part(item)
+ if part is not None:
+ parts.append(part)
+ tc = (
+ parts
+ if any(p["type"] == "image_url" for p in parts)
+ else " ".join(p["text"] for p in parts)
)
tool_results.append(
{
"role": "tool",
"tool_call_id": b["tool_use_id"],
- "content": str(tc),
+ "content": tc if isinstance(tc, list) else str(tc),
}
)
@@ -218,12 +244,21 @@ def fold_tool_results_into_user(messages: list[dict]) -> list[dict]:
response["content"] = msg.get("content", "")
if tool_call_id:
response["tool_call_id"] = tool_call_id
- out.append(
- {
- "role": "user",
- "content": json.dumps({"tool_response": response}, indent = 2),
- }
- )
+ content = response["content"]
+ images = []
+ if isinstance(content, list):
+ images = [p for p in content if isinstance(p, dict) and p.get("type") == "image_url"]
+ if images:
+ # Images must be real parts; the rest of the result stays inside the tool_response
+ # wrapper, so tool text never reads as user text and the archive still matches it.
+ response["content"] = [p for p in content if p not in images]
+ folded_content = [
+ {"type": "text", "text": json.dumps({"tool_response": response}, indent = 2)},
+ *images,
+ ]
+ else:
+ folded_content = json.dumps({"tool_response": response}, indent = 2)
+ out.append({"role": "user", "content": folded_content})
return out
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 43c10eadc13..68ea3e191e9 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1938,6 +1938,9 @@ def _openai_llama_admission_image_tokens(llama_backend) -> int:
Over-reserving is the safe direction; under-reserving hands out a slot the cache
cannot back.
"""
+ # A text-only model is sent no image embeddings.
+ if getattr(llama_backend, "is_vision", True) is False:
+ return 0
projector = getattr(llama_backend, "_mmproj_projector_type", None)
known = _MMPROJ_IMAGE_TOKEN_MAX.get(str(projector).strip().lower()) if projector else None
cap = max(
@@ -1997,17 +2000,19 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],
continue
part_type = part.get("type")
- # The same filter anthropic_messages_to_openai applies, so the estimate
- # charges what that sends. The content list is untyped, so a screenshot an
- # agent's tool returned, a document and a future block type all arrive
- # here, and none of them reach the wire to earn an allowance.
+ # Match the native tool-result conversion: text and image blocks reach the wire.
if part_type == "tool_result" and isinstance(part.get("content"), list):
part = dict(part)
- part["content"] = [
- block
- for block in part["content"]
- if isinstance(block, dict) and block.get("type") == "text"
- ]
+ tool_content = []
+ for block in part["content"]:
+ if not isinstance(block, dict):
+ continue
+ if block.get("type") == "text":
+ tool_content.append(block)
+ elif block.get("type") == "image":
+ image_parts += 1
+ tool_content.append(_openai_llama_admission_compact_image_part(block))
+ part["content"] = tool_content
estimate_content.append(part)
continue
@@ -7876,39 +7881,42 @@ def _request_has_image(payload) -> bool:
return _messages_have_image(payload.messages)
-def _anthropic_request_has_image(payload) -> bool:
- # Mirror anthropic_messages_to_openai: an Anthropic image block carries
- # ``type == "image"`` (typed AnthropicImageBlock or a raw dict).
- for msg in getattr(payload, "messages", None) or []:
- content = getattr(msg, "content", None)
+def _anthropic_image_blocks(payload, *, tool_results: bool = True):
+ for msg in getattr(payload, "messages", None) or ():
+ content = msg.get("content") if isinstance(msg, dict) else msg.content
if not isinstance(content, list):
continue
for block in content:
- bt = block.get("type") if isinstance(block, dict) else getattr(block, "type", None)
- if bt == "image":
- return True
- return False
+ block = block if isinstance(block, dict) else block.model_dump()
+ if block.get("type") == "image":
+ yield block
+ elif (
+ tool_results
+ and block.get("type") == "tool_result"
+ and isinstance(block.get("content"), list)
+ ):
+ for part in block["content"]:
+ if isinstance(part, dict) and part.get("type") == "image":
+ yield part
+
+
+def _anthropic_request_has_image(payload, *, tool_results: bool = True) -> bool:
+ return next(_anthropic_image_blocks(payload, tool_results = tool_results), None) is not None
def _anthropic_local_image_payloads(payload) -> list[str]:
- """Base64 image sources translated by the Anthropic endpoint."""
encoded_images = []
- for msg in getattr(payload, "messages", None) or ():
- content = msg.get("content") if isinstance(msg, dict) else msg.content
- if not isinstance(content, list):
+ for block in _anthropic_image_blocks(payload):
+ source = block.get("source")
+ if not isinstance(source, dict):
continue
- for block in content:
- block_type = block.get("type") if isinstance(block, dict) else block.type
- if block_type != "image":
- continue
- source = block.get("source") if isinstance(block, dict) else block.source
- source_type = source.get("type") if isinstance(source, dict) else source.type
- data = source.get("data") if isinstance(source, dict) else source.data
- if source_type == "base64" and isinstance(data, str):
- encoded_images.append(data)
- url = source.get("url") if isinstance(source, dict) else source.url
- if source_type == "url" and isinstance(url, str) and url.startswith("data:"):
- encoded_images.append(url.partition(",")[2])
+ source_type = source.get("type")
+ data = source.get("data")
+ if source_type == "base64" and isinstance(data, str):
+ encoded_images.append(data)
+ url = source.get("url")
+ if source_type == "url" and isinstance(url, str) and url.startswith("data:"):
+ encoded_images.append(url.partition(",")[2])
return encoded_images
@@ -30685,7 +30693,7 @@ async def anthropic_count_tokens(
_switch_model_for_payload(payload),
request,
current_subject,
- require_vision = _anthropic_request_has_image(payload),
+ require_vision = _anthropic_request_has_image(payload, tool_results = False),
# count_tokens only tokenizes (no generation), so it must not adopt the resident
# model; the middleware likewise excludes count_tokens from its claim.
claim_resident = False,
@@ -30708,6 +30716,7 @@ async def anthropic_count_tokens(
[m.model_dump() for m in payload.messages],
payload.system,
preserve_thinking = _anthropic_preserve_thinking(llama_backend, payload),
+ tool_result_images = llama_backend.is_vision,
)
# Apply the same sanitization /messages does before generation, so the count
# matches the prompt the real request would build (otherwise empty-assistant
@@ -30726,7 +30735,7 @@ async def anthropic_count_tokens(
_count_server_tools = (
_anthropic_selects_server_tools(payload, _count_studio_tools, _count_has_client_tool)
and llama_backend.supports_tools
- and not _anthropic_request_has_image(payload)
+ and not _anthropic_request_has_image(payload, tool_results = llama_backend.is_vision)
)
_count_openai_client_tools = [
tool
@@ -30776,7 +30785,16 @@ async def anthropic_count_tokens(
status_code = 503,
detail = "Unable to count tokens with the loaded model tokenizer.",
)
- return JSONResponse(content = {"input_tokens": int(count)})
+ # /apply-template renders an image as a marker, not its projector embeddings, so charge
+ # the same per-image allowance admission reserves.
+ image_parts = sum(
+ isinstance(part, dict) and part.get("type") == "image_url"
+ for message in openai_messages
+ if isinstance(message.get("content"), list)
+ for part in message["content"]
+ )
+ image_tokens = image_parts * _openai_llama_admission_image_tokens(llama_backend)
+ return JSONResponse(content = {"input_tokens": int(count) + image_tokens})
def _set_or_prepend_system_message(
@@ -30838,6 +30856,9 @@ async def anthropic_messages(
_validate_anthropic_client_tools(payload.tools)
_anthropic_has_image = _anthropic_request_has_image(payload)
+ # Tool-result images become a note on a text-only model, so until the backend is known
+ # only top-level images require vision or rule out server tools.
+ _anthropic_top_level_image = _anthropic_request_has_image(payload, tool_results = False)
_anthropic_image_b64s = _anthropic_local_image_payloads(payload)
# Mixing Anthropic server tools with custom client tools is unsupported (the
@@ -30883,7 +30904,7 @@ async def anthropic_messages(
_selects_server_tools = _anthropic_selects_server_tools(
payload, requested_studio_tools, _has_client_tool
)
- _server_tools_requested_pre = _selects_server_tools and not _anthropic_has_image
+ _server_tools_requested_pre = _selects_server_tools and not _anthropic_top_level_image
if _server_tools_requested_pre:
from core.inference.tools import ALL_TOOLS as _ALL_TOOLS_PRE
@@ -30923,7 +30944,7 @@ async def anthropic_messages(
_switch_model_for_payload(payload),
request,
current_subject,
- require_vision = _anthropic_has_image,
+ require_vision = _anthropic_top_level_image,
# The image normalization below can still 400 after this switch, so defer the claim:
# the middleware claims on a 2xx, so a rejected request never strands a preview-owned
# model.
@@ -30958,6 +30979,7 @@ async def anthropic_messages(
[m.model_dump() for m in payload.messages],
payload.system,
preserve_thinking = _anthropic_preserve_thinking(llama_backend, payload),
+ tool_result_images = llama_backend.is_vision,
)
# Strip synthetic provider-side builtin tool history (web_search,
# web_fetch, code_execution, image_generation cards tagged with
diff --git a/studio/backend/tests/test_anthropic_native_tool_images.py b/studio/backend/tests/test_anthropic_native_tool_images.py
new file mode 100644
index 00000000000..6802769028b
--- /dev/null
+++ b/studio/backend/tests/test_anthropic_native_tool_images.py
@@ -0,0 +1,192 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+
+import base64
+import copy
+import json
+from io import BytesIO
+
+import httpx
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+from PIL import Image
+
+from core.inference.anthropic_compat import (
+ TOOL_RESULT_IMAGE_OMITTED,
+ anthropic_messages_to_openai,
+ fold_tool_results_into_user,
+)
+from models.inference import AnthropicMessagesRequest
+from routes import inference as inf
+from studio.backend.tests.test_anthropic_messages import _mock_backend
+
+
+def image_block():
+ buffer = BytesIO()
+ Image.new("RGB", (2, 2), "red").save(buffer, format = "WEBP")
+ return {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/webp",
+ "data": base64.b64encode(buffer.getvalue()).decode(),
+ },
+ }
+
+
+def payload(parts):
+ return {
+ "model": "vision-test",
+ "max_tokens": 16,
+ "tools": [{"name": "capture", "input_schema": {"type": "object"}}],
+ "messages": [
+ {"role": "user", "content": "Inspect these captures."},
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "tool_use", "id": ident, "name": "capture", "input": {}}
+ for ident in ("toolu_first", "toolu_second")
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {"type": "tool_result", "tool_use_id": "toolu_first", "content": parts},
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_second",
+ "content": "second result",
+ },
+ {"type": "text", "text": "Compare them."},
+ ],
+ },
+ ],
+ }
+
+
+ "order", [("image",), ("text", "image"), ("image", "text"), ("text", "image", "text")]
+)
+def test_native_images_keep_order_and_tool_identity(order):
+ parts = [
+ image_block() if kind == "image" else {"type": "text", "text": str(i)}
+ for i, kind in enumerate(order)
+ ]
+ request = AnthropicMessagesRequest(**payload(parts))
+ converted = anthropic_messages_to_openai([m.model_dump() for m in request.messages])
+ assert [m.get("tool_call_id") for m in converted if m["role"] == "tool"] == [
+ "toolu_first",
+ "toolu_second",
+ ]
+ assert [p["type"] for p in converted[2]["content"]] == [
+ "image_url" if k == "image" else k for k in order
+ ]
+ assert converted[3]["content"] == "second result"
+ assert converted[-1]["content"] == "Compare them."
+ assert inf._anthropic_request_has_image(request)
+ assert inf._anthropic_local_image_payloads(request) == [
+ p["source"]["data"] for p in parts if p["type"] == "image"
+ ]
+ folded = fold_tool_results_into_user(converted)
+ wrapper = json.loads(folded[2]["content"][0]["text"])["tool_response"]
+ assert wrapper["tool_call_id"] == "toolu_first"
+ assert wrapper["content"] == [p for p in converted[2]["content"] if p["type"] == "text"]
+ assert [p["type"] for p in folded[2]["content"][1:]] == ["image_url"] * order.count("image")
+
+
+ "source", ["invalid", {"type": "url", "url": 5}, {"type": "base64", "data": 5}]
+)
+def test_malformed_tool_result_image_source_is_skipped(source):
+ request = AnthropicMessagesRequest(**payload([{"type": "image", "source": source}]))
+ converted = anthropic_messages_to_openai([m.model_dump() for m in request.messages])
+ assert converted[2]["content"] == ""
+ assert inf._anthropic_local_image_payloads(request) == []
+
+
[email protected]("vision", [True, False])
+def test_native_image_http_generation_and_count(monkeypatch, vision):
+ seen = {}
+
+ async def switch(*args, **kwargs):
+ seen.setdefault("preflight", []).append(kwargs)
+
+ def count(messages, *args, **kwargs):
+ seen["count"] = copy.deepcopy(messages)
+ return 42
+
+ _mock_backend(
+ monkeypatch,
+ is_vision = vision,
+ supports_tool_passthrough = True,
+ base_url = "http://llama.test",
+ effective_parallel_slots = 4,
+ count_chat_tokens = count,
+ )
+ monkeypatch.setattr(inf, "_maybe_auto_switch_model", switch)
+ real_client = httpx.AsyncClient
+
+ def capture(request):
+ seen["wire"] = json.loads(request.content)
+ return httpx.Response(
+ 200,
+ json = {
+ "choices": [
+ {
+ "message": {"role": "assistant", "content": "The image is red."},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 42, "completion_tokens": 5},
+ },
+ )
+
+ monkeypatch.setattr(
+ inf.httpx,
+ "AsyncClient",
+ lambda **kwargs: real_client(transport = httpx.MockTransport(capture)),
+ )
+ app = FastAPI()
+ app.include_router(inf.router, prefix = "/v1")
+ app.dependency_overrides[inf.get_current_subject] = lambda: "test"
+ body = payload([{"type": "text", "text": "capture"}, image_block()])
+ with TestClient(app) as client:
+ response = client.post("/v1/messages", json = body)
+ counted = client.post("/v1/messages/count_tokens", json = body)
+ assert response.status_code == 200, response.text
+ assert counted.status_code == 200, counted.text
+ image_tokens = inf._OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS if vision else 0
+ assert counted.json()["input_tokens"] == 42 + image_tokens
+ assert response.json()["content"][0]["text"] == "The image is red."
+ assert [p["require_vision"] for p in seen["preflight"]] == [False, False]
+ assert len(seen["preflight"][0]["image_preflight"]["b64s"]) == 1
+ sent = seen["wire"]["messages"][2]["content"]
+ counted_content = seen["count"][2]["content"]
+ if vision:
+ assert [p["type"] for p in counted_content] == ["text", "image_url"]
+ url = sent[1]["image_url"]["url"]
+ assert url.startswith("data:image/png;base64,")
+ assert Image.open(BytesIO(base64.b64decode(url.split(",", 1)[1]))).size == (2, 2)
+ else:
+ assert sent == counted_content == f"capture {TOOL_RESULT_IMAGE_OMITTED}"
+
+
+def test_text_only_tool_image_keeps_the_server_tool_permission_gate(monkeypatch):
+ switched = []
+
+ async def switch(*args, **kwargs):
+ switched.append(kwargs)
+
+ backend = _mock_backend(monkeypatch, is_vision = False)
+ monkeypatch.setattr(inf, "_maybe_auto_switch_model", switch)
+ body = payload([{"type": "text", "text": "capture"}, image_block()])
+ del body["tools"]
+ body.update(enable_tools = True, permission_mode = "ask")
+ app = FastAPI()
+ app.include_router(inf.router, prefix = "/v1")
+ app.dependency_overrides[inf.get_current_subject] = lambda: "test"
+ with TestClient(app) as client:
+ response = client.post("/v1/messages", json = body)
+ assert response.status_code == 400, response.text
+ assert "permission_mode" in response.text
+ assert switched == [] and backend.calls == []
diff --git a/studio/backend/tests/test_conversation_archive.py b/studio/backend/tests/test_conversation_archive.py
index 40dc79bebfd..7e5d7f4ed56 100644
--- a/studio/backend/tests/test_conversation_archive.py
+++ b/studio/backend/tests/test_conversation_archive.py
@@ -635,6 +635,12 @@ def test_a_folded_retrieval_result_is_still_kept_out_of_the_archive():
assert "ASKEDAFTERWARDS?" in rendered
assert "ZQXVARA123" in rendered
+ recalled[2]["content"] = [{"type": "text", "text": "<chunk>RETRIEVEDPASSAGE</chunk>"}]
+ listed = _coalesce_consecutive_user_turns(fold_tool_results_into_user(recalled))
+ rendered = archive.render_turn(archive._archivable(listed))
+ assert "RETRIEVEDPASSAGE" not in rendered
+ assert "ASKEDAFTERWARDS?" in rendered
+
# An image on the next question makes the coalesce produce a part list, not a string, and
# reading only strings archived the passage on exactly the turns that carry an image.
with_image = [
diff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
index 903f5772306..bf4537cc0fe 100644
--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
@@ -15,6 +15,9 @@ estimate for a lease that runs up to 25 growing rounds.
"""
import base64
+import copy
+
+import pytest
from models.inference import AnthropicMessagesRequest
from routes.inference import (
@@ -491,14 +494,8 @@ class TestAToolResultScreenshotIsNotPricedByItsBase64:
)
def test_the_charge_matches_what_the_translation_actually_sends(self):
- """The two halves have to agree, so this fails on whichever side moves first.
-
- `anthropic_messages_to_openai` keeps only the text blocks of a list
- `tool_result`, so the nested image never reaches llama-server and earns no mtmd
- allowance. Start forwarding it and this fails, which is the reminder that
- admission has to start charging for it.
- """
- data = _image_b64(64)
+ """Every forwarded tool image earns an embedding allowance, not a base64 charge."""
+ data = _TINY_PNG
payload = self._request(data)
estimate_messages, image_parts = _openai_llama_admission_messages_for_estimate(
@@ -515,6 +512,44 @@ class TestAToolResultScreenshotIsNotPricedByItsBase64:
f"sends the image (forwarded={forwarded}, image_parts={image_parts})"
)
+ from types import SimpleNamespace
+
+ from routes.inference import _openai_llama_admission_image_tokens
+
+ text_only = anthropic_messages_to_openai(
+ [message.model_dump() for message in payload.messages], None, tool_result_images = False
+ )
+ assert data not in str(text_only)
+ assert _openai_llama_admission_image_tokens(SimpleNamespace(is_vision = False)) == 0
+
+ @pytest.mark.parametrize("source_type", ["base64", "url"])
+ @pytest.mark.parametrize("with_text", [False, True])
+ def test_each_nested_image_is_compacted_and_charged(self, source_type, with_text):
+ payload = self._request(_TINY_PNG)
+ blocks = payload.messages[-1].content[0].content
+ first = blocks[-1]
+ second = copy.deepcopy(first)
+ second["source"]["data"] = _OTHER_PNG
+ if source_type == "url":
+ for block in (first, second):
+ block["source"] = {
+ "type": "url",
+ "url": f"data:image/png;base64,{block['source']['data']}",
+ }
+ blocks[:] = [first, *blocks[:1], second] if with_text else [first, second]
+ original = payload.model_dump()
+
+ compact, count = _openai_llama_admission_messages_for_estimate(payload.messages)
+ sent = anthropic_messages_to_openai([m.model_dump() for m in payload.messages])
+ forwarded = [p for m in sent if m["role"] == "tool" for p in m["content"]]
+ assert count == sum(p["type"] == "image_url" for p in forwarded) == 2
+ assert _TINY_PNG not in str(compact) and _OTHER_PNG not in str(compact)
+ assert payload.model_dump() == original
+ assert (
+ _openai_llama_admission_tokens(payload, budget = 1_000_000, capacity = 4)
+ >= 2 * _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS
+ )
+
class TestEveryBlockTheTranslationDropsIsPricedTheSameWay:
"""`tool_result` content is an untyped list, so an image is only one of the block types
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 4ca7e3821a0..1dd7750b268 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -4384,6 +4384,8 @@ def test_messages_have_image_helper():
def test_anthropic_request_has_image_helper():
+ from models.inference import AnthropicImageBlock
+
f = inference_route._anthropic_request_has_image
text = SimpleNamespace(messages = [SimpleNamespace(content = "hi")])
assert f(text) is False
@@ -4393,7 +4395,18 @@ def test_anthropic_request_has_image_helper():
assert f(text_block) is False
dict_img = SimpleNamespace(messages = [SimpleNamespace(content = [{"type": "image"}])])
assert f(dict_img) is True
- typed_img = SimpleNamespace(messages = [SimpleNamespace(content = [SimpleNamespace(type = "image")])])
+ typed_img = SimpleNamespace(
+ messages = [
+ SimpleNamespace(
+ content = [
+ AnthropicImageBlock(
+ type = "image",
+ source = {"type": "base64", "media_type": "image/png", "data": "AAAA"},
+ )
+ ]
+ )
+ ]
+ )
assert f(typed_img) is True
@@ -4406,12 +4419,13 @@ def test_responses_and_anthropic_wire_require_vision_from_images():
assert "_responses_has_image = _messages_have_image(" in responses_src
assert "require_vision = _responses_has_image" in responses_src
anthropic_src = inspect.getsource(inference_route.anthropic_messages)
- assert "_anthropic_has_image = _anthropic_request_has_image(" in anthropic_src
- assert "require_vision = _anthropic_has_image" in anthropic_src
+ guard = "_anthropic_request_has_image(payload, tool_results = False)"
+ assert f"_anthropic_top_level_image = {guard}" in anthropic_src
+ assert "require_vision = _anthropic_top_level_image" in anthropic_src
# /messages/count_tokens shares the /messages translation, so it needs the same
# guard: an image count must not evict a vision model for a text-only target.
count_src = inspect.getsource(inference_route.anthropic_count_tokens)
- assert "require_vision = _anthropic_request_has_image(" in count_src
+ assert f"require_vision = {guard}" in count_src
# ── codex review (round 5): count_tokens tools, tool_choice, process-wide gate ──
@@ -4450,7 +4464,7 @@ def test_count_tokens_forwards_vision_guard_to_switch(monkeypatch):
captured["gguf_only"] = gguf_only
raise _Reached()
- monkeypatch.setattr(inference_route, "_anthropic_request_has_image", lambda p: True)
+ monkeypatch.setattr(inference_route, "_anthropic_request_has_image", lambda p, **_: True)
monkeypatch.setattr(inference_route, "_maybe_auto_switch_model", _capture)
payload = _anthropic_payload_with_tools(None) # no tools -> tool validation passes
with pytest.raises(_Reached):