Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: pass search results and documents through to the model on /v1/messages (#11152)
Nilay3781136fd2fe
diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.pyindex d7e83d8a039..77efdc57cb7 100644--- a/studio/backend/core/inference/anthropic_compat.py+++ b/studio/backend/core/inference/anthropic_compat.py@@ -41,6 +41,47 @@ def anthropic_tool_use_id(upstream_id = None) -> str:TOOL_RESULT_IMAGE_OMITTED = "[image omitted: this model cannot view images]"+DOCUMENT_OMITTED = "[document omitted: only text documents can be read]"+DOCUMENT_IMAGE_OMITTED = "[image omitted: images inside documents are not read]"+++def _anthropic_block_texts(content: Any) -> list[str]:+ if isinstance(content, str):+ return [content]+ if not isinstance(content, list):+ return []+ texts = []+ for b in content:+ if not isinstance(b, dict):+ continue+ if b.get("type") == "text" and isinstance(b.get("text"), str):+ texts.append(b["text"])+ elif b.get("type") == "image":+ texts.append(DOCUMENT_IMAGE_OMITTED)+ return texts+++def anthropic_reference_block_text(block: dict) -> str:+ btype = block.get("type")+ source = block.get("source")+ if btype == "search_result":+ labels = {"Title": block.get("title"), "Source": source}+ body = _anthropic_block_texts(block.get("content"))+ elif btype == "document":+ labels = {"Title": block.get("title"), "Context": block.get("context")}+ stype = source.get("type") if isinstance(source, dict) else None+ if stype == "text" and isinstance(source.get("data"), str):+ body = [source["data"]]+ elif stype == "content":+ body = _anthropic_block_texts(source.get("content"))+ else:+ body = [DOCUMENT_OMITTED]+ else:+ return ""+ header = [+ f"{label}: {value}" for label, value in labels.items() if isinstance(value, str) and value+ ]+ return "\n".join([*header, *body])def _anthropic_image_block_to_openai_part(block: dict) -> Optional[dict]:@@ -163,6 +204,8 @@ def anthropic_messages_to_openai(btype = b.get("type", "")if btype == "text":user_parts.append({"type": "text", "text": b["text"]})+ elif reference := anthropic_reference_block_text(b):+ user_parts.append({"type": "text", "text": reference})elif btype == "image":part = _anthropic_image_block_to_openai_part(b)if part is not None:@@ -177,6 +220,8 @@ def anthropic_messages_to_openai(continueif item.get("type") == "text":parts.append({"type": "text", "text": item["text"]})+ elif reference := anthropic_reference_block_text(item):+ parts.append({"type": "text", "text": reference})elif item.get("type") == "image":if not tool_result_images:parts.append(@@ -189,7 +234,7 @@ def anthropic_messages_to_openai(tc = (partsif any(p["type"] == "image_url" for p in parts)- else " ".join(p["text"] for p in parts)+ else "\n".join(p["text"] for p in parts))tool_results.append({diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex b77bcbe6c43..f12302975b9 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -3296,15 +3296,17 @@ class AnthropicToolResultBlock(BaseModel):return "" if v is None else v-# Block types the converter translates explicitly. Anything else (thinking / redacted_thinking, a-# provider block a resumed session replays, or a future type) is accepted as an unknown block and-# dropped by the converter, rather than 400-ing the whole request on strict validation.+# Block types with typed models. Anything else (a search_result or document, a provider block a+# resumed session replays, or a future type) is accepted as an unknown block, which the converter+# renders if it can and otherwise drops, rather than 400-ing the whole request on strict validation._KNOWN_ANTHROPIC_BLOCK_TYPES = frozenset({"text", "image", "tool_use", "tool_result", "thinking", "redacted_thinking"})# Thinking blocks are replayed only in assistant turns; the converter drops them# from user content, so accepting them there would silently lose a user turn.-_USER_ANTHROPIC_BLOCK_TYPES = frozenset({"text", "image", "tool_use", "tool_result"})+_USER_ANTHROPIC_BLOCK_TYPES = frozenset(+ {"text", "image", "tool_use", "tool_result", "search_result", "document"}+)class AnthropicUnknownBlock(BaseModel):@@ -3417,6 +3419,20 @@ class AnthropicMessage(BaseModel):# value would raise TypeError, escaping as a 500 instead of a clean 400.if not isinstance(btype, str) or btype not in _USER_ANTHROPIC_BLOCK_TYPES:raise ValueError(f"unsupported content block type {btype!r} in a user message")+ # A PDF, url or file document cannot be read. Only inside a tool result, which+ # clients resend with history, does it degrade to a note instead of a 400.+ if btype == "document":+ source = (+ block.get("source")+ if isinstance(block, dict)+ else getattr(block, "source", None)+ )+ stype = source.get("type") if isinstance(source, dict) else None+ if stype not in ("text", "content"):+ raise ValueError(+ f"unsupported document source type {stype!r}: only text and content "+ "documents can be read"+ )return datadiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex c1e2c2b548f..918cdaeb8a0 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -2040,7 +2040,11 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],continuepart_type = part.get("type")- # Match the native tool-result conversion: text and image blocks reach the wire.+ if reference := anthropic_reference_block_text(part):+ estimate_content.append({"type": "text", "text": reference})+ continue++ # Match the native tool-result conversion: only the blocks it renders reach the wire.if part_type == "tool_result" and isinstance(part.get("content"), list):part = dict(part)tool_content = []@@ -2049,6 +2053,8 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],continueif block.get("type") == "text":tool_content.append(block)+ elif reference := anthropic_reference_block_text(block):+ tool_content.append({"type": "text", "text": reference})elif block.get("type") == "image":image_parts += 1tool_content.append(_openai_llama_admission_compact_image_part(block))@@ -3296,6 +3302,7 @@ from models.inference import ()from core.inference.anthropic_compat import (anthropic_messages_to_openai,+ anthropic_reference_block_text,fold_tool_results_into_user,anthropic_schema_client_tool_kind,anthropic_tools_to_openai,diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.pyindex c55cb649c0d..1133b8d56f6 100644--- a/studio/backend/tests/test_anthropic_messages.py+++ b/studio/backend/tests/test_anthropic_messages.py@@ -28,6 +28,8 @@ from models.inference import (AnthropicResponseToolUseBlock,)from core.inference.anthropic_compat import (+ DOCUMENT_IMAGE_OMITTED,+ DOCUMENT_OMITTED,anthropic_messages_to_openai,anthropic_schema_client_tool_kind,anthropic_tools_to_openai,@@ -1028,7 +1030,137 @@ class TestAnthropicMessagesToOpenAI:}]result = anthropic_messages_to_openai(msgs)- assert result[0]["content"] == "Line 1 Line 2"+ assert result[0]["content"] == "Line 1\nLine 2"++ def test_tool_result_search_results_keep_title_source_and_text(self):+ msgs = [+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "tu_1",+ "content": [+ {+ "type": "search_result",+ "source": "https://docs.example.com/vault",+ "title": "Vault",+ "content": [+ {"type": "text", "text": "The code is PURPLE-ELEPHANT-42."},+ {"type": "text", "text": "Rotate it monthly."},+ ],+ "citations": {"enabled": True},+ },+ {+ "type": "search_result",+ "source": "kb://faq",+ "title": "FAQ",+ "content": [{"type": "text", "text": "Ask the admin."}],+ },+ ],+ },+ ],+ }+ ]+ result = anthropic_messages_to_openai(msgs)+ assert result == [+ {+ "role": "tool",+ "tool_call_id": "tu_1",+ "content": "Title: Vault\nSource: https://docs.example.com/vault\n"+ "The code is PURPLE-ELEPHANT-42.\nRotate it monthly.\n"+ "Title: FAQ\nSource: kb://faq\nAsk the admin.",+ }+ ]++ @pytest.mark.parametrize(+ "source, body",+ [+ ({"type": "text", "media_type": "text/plain", "data": "Plain body."}, "Plain body."),+ (+ {+ "type": "content",+ "content": [+ {"type": "text", "text": "First chunk"},+ {"type": "text", "text": "Second chunk"},+ ],+ },+ "First chunk\nSecond chunk",+ ),+ ({"type": "content", "content": "Whole body."}, "Whole body."),+ (+ {+ "type": "content",+ "content": [+ {"type": "text", "text": "Before"},+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": "iVBOR",+ },+ },+ {"type": "text", "text": "After"},+ ],+ },+ f"Before\n{DOCUMENT_IMAGE_OMITTED}\nAfter",+ ),+ (+ {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0="},+ DOCUMENT_OMITTED,+ ),+ ({"type": "url", "url": "https://example.com/a.pdf"}, DOCUMENT_OMITTED),+ ],+ )+ def test_tool_result_document_renders_its_readable_source(self, source, body):+ document = {+ "type": "document",+ "source": source,+ "title": "Handbook",+ "context": "Internal",+ }+ msgs = [+ {+ "role": "user",+ "content": [{"type": "tool_result", "tool_use_id": "tu_1", "content": [document]}],+ }+ ]+ result = anthropic_messages_to_openai(msgs)+ assert result[0]["content"] == f"Title: Handbook\nContext: Internal\n{body}"++ def test_top_level_search_result_and_document_reach_the_user_turn(self):+ request = AnthropicMessagesRequest(+ model = "x",+ max_tokens = 16,+ messages = [+ {+ "role": "user",+ "content": [+ {+ "type": "search_result",+ "source": "kb://vault",+ "title": "Vault",+ "content": [{"type": "text", "text": "PURPLE-ELEPHANT-42"}],+ },+ {+ "type": "document",+ "source": {"type": "text", "media_type": "text/plain", "data": "Memo."},+ },+ {"type": "document", "source": {"type": "content", "content": "Notes."}},+ {"type": "text", "text": "What is the code?"},+ ],+ }+ ],+ )+ result = anthropic_messages_to_openai([m.model_dump() for m in request.messages])+ assert result == [+ {+ "role": "user",+ "content": "Title: Vault\nSource: kb://vault\nPURPLE-ELEPHANT-42\n"+ "Memo.\nNotes.\nWhat is the code?",+ }+ ]def test_image_base64_block_becomes_multimodal_part(self):msgs = [@@ -3889,11 +4021,41 @@ def test_user_unknown_block_rejected_not_silently_dropped():model = "x",max_tokens = 16,messages = [- {"role": "user", "content": [{"type": "document", "source": {}}]},+ {"role": "user", "content": [{"type": "container_upload", "file_id": "f"}]},],)+ "source",+ [+ {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0="},+ {"type": "url", "url": "https://example.com/a.pdf"},+ {"type": "file", "file_id": "file_1"},+ ],+)+def test_user_unreadable_document_rejected_outside_tool_results(source):+ from pydantic import ValidationError++ document = {"type": "document", "source": source}+ with pytest.raises(ValidationError, match = "unsupported document source type"):+ AnthropicMessagesRequest(+ model = "x",+ max_tokens = 16,+ messages = [{"role": "user", "content": [{"type": "text", "text": "Read"}, document]}],+ )+ AnthropicMessagesRequest(+ model = "x",+ max_tokens = 16,+ messages = [+ {+ "role": "user",+ "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [document]}],+ }+ ],+ )++def test_user_translatable_blocks_still_accepted():# text / image / tool_result are translatable, so a real user message built from# them must still pass; the unknown-block guard only trips on other types.diff --git a/studio/backend/tests/test_anthropic_native_tool_images.py b/studio/backend/tests/test_anthropic_native_tool_images.pyindex 6802769028b..0d84d89fc48 100644--- a/studio/backend/tests/test_anthropic_native_tool_images.py+++ b/studio/backend/tests/test_anthropic_native_tool_images.py@@ -168,7 +168,7 @@ def test_native_image_http_generation_and_count(monkeypatch, vision):assert url.startswith("data:image/png;base64,")assert Image.open(BytesIO(base64.b64decode(url.split(",", 1)[1]))).size == (2, 2)else:- assert sent == counted_content == f"capture {TOOL_RESULT_IMAGE_OMITTED}"+ assert sent == counted_content == f"capture\n{TOOL_RESULT_IMAGE_OMITTED}"def test_text_only_tool_image_keeps_the_server_tool_permission_gate(monkeypatch):diff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.pyindex d409e4000a1..338303fd3ab 100644--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py@@ -553,8 +553,8 @@ class TestAToolResultScreenshotIsNotPricedByItsBase64:class TestEveryBlockTheTranslationDropsIsPricedTheSameWay:"""`tool_result` content is an untyped list, so an image is only one of the block types- that reach it. A document, a search result and a nested `tool_result` are dropped by- the same translation filter, and each was charged its base64 as prompt text -- the+ that reach it. A PDF document, a malformed search result and a nested `tool_result` send+ at most a short note, and each was charged its base64 as prompt text -- thewhole of a 32768-token cache for a request that sends a couple of hundred characters."""@@ -641,6 +641,41 @@ class TestEveryBlockTheTranslationDropsIsPricedTheSameWay:estimate_messages), f"{where}: the text beside it IS sent, so dropping it under-reserves"+ def test_search_result_and_document_text_is_charged_where_it_is_sent(self):+ search_result = {+ "type": "search_result",+ "source": "kb://vault",+ "title": "Vault",+ "content": [{"type": "text", "text": "PURPLE-ELEPHANT-42 " * 200}],+ }+ document = {+ "type": "document",+ "source": {"type": "text", "media_type": "text/plain", "data": "MEMO-BODY " * 200},+ "title": "Memo",+ }+ for block, header in (+ (search_result, "Title: Vault\nSource: kb://vault\n"),+ (document, "Title: Memo\n"),+ ):+ for payload in (+ self._request(block, text_first = True),+ AnthropicMessagesRequest(+ model = "default",+ max_tokens = 128,+ messages = [{"role": "user", "content": [block]}],+ ),+ ):+ sent = anthropic_messages_to_openai([m.model_dump() for m in payload.messages])+ rendered = sent[-1]["content"].rsplit("the tool answered\n", 1)[-1]+ estimate_messages, image_parts = _openai_llama_admission_messages_for_estimate(+ payload.messages+ )+ assert rendered.startswith(header)+ assert rendered in str(estimate_messages).replace("\\n", "\n")+ assert image_parts == 0+ cost = _openai_llama_admission_tokens(payload, budget = 1_000_000, capacity = 4)+ assert cost > len(rendered) // 8+def test_a_tool_result_the_translation_does_forward_is_still_charged(self):"""The other side of the boundary: string `tool_result` content is forwardedverbatim, base64-looking text included, so it keeps costing what its length costs