Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: pass search results and documents through to the model on /v1/messages (#11152)

Nilay3781136fd2fe
diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py
index d7e83d8a039..77efdc57cb7 100644
--- a/studio/backend/core/inference/anthropic_compat.py
+++ b/studio/backend/core/inference/anthropic_compat.py
@@ -41,6 +41,47 @@ def anthropic_tool_use_id(upstream_id = None) -> str:
TOOL_RESULT_IMAGE_OMITTED = "[image omitted: this model cannot view images]"
+DOCUMENT_OMITTED = "[document omitted: only text documents can be read]"
+DOCUMENT_IMAGE_OMITTED = "[image omitted: images inside documents are not read]"
+
+
+def _anthropic_block_texts(content: Any) -> list[str]:
+ if isinstance(content, str):
+ return [content]
+ if not isinstance(content, list):
+ return []
+ texts = []
+ for b in content:
+ if not isinstance(b, dict):
+ continue
+ if b.get("type") == "text" and isinstance(b.get("text"), str):
+ texts.append(b["text"])
+ elif b.get("type") == "image":
+ texts.append(DOCUMENT_IMAGE_OMITTED)
+ return texts
+
+
+def anthropic_reference_block_text(block: dict) -> str:
+ btype = block.get("type")
+ source = block.get("source")
+ if btype == "search_result":
+ labels = {"Title": block.get("title"), "Source": source}
+ body = _anthropic_block_texts(block.get("content"))
+ elif btype == "document":
+ labels = {"Title": block.get("title"), "Context": block.get("context")}
+ stype = source.get("type") if isinstance(source, dict) else None
+ if stype == "text" and isinstance(source.get("data"), str):
+ body = [source["data"]]
+ elif stype == "content":
+ body = _anthropic_block_texts(source.get("content"))
+ else:
+ body = [DOCUMENT_OMITTED]
+ else:
+ return ""
+ header = [
+ f"{label}: {value}" for label, value in labels.items() if isinstance(value, str) and value
+ ]
+ return "\n".join([*header, *body])
def _anthropic_image_block_to_openai_part(block: dict) -> Optional[dict]:
@@ -163,6 +204,8 @@ def anthropic_messages_to_openai(
btype = b.get("type", "")
if btype == "text":
user_parts.append({"type": "text", "text": b["text"]})
+ elif reference := anthropic_reference_block_text(b):
+ user_parts.append({"type": "text", "text": reference})
elif btype == "image":
part = _anthropic_image_block_to_openai_part(b)
if part is not None:
@@ -177,6 +220,8 @@ def anthropic_messages_to_openai(
continue
if item.get("type") == "text":
parts.append({"type": "text", "text": item["text"]})
+ elif reference := anthropic_reference_block_text(item):
+ parts.append({"type": "text", "text": reference})
elif item.get("type") == "image":
if not tool_result_images:
parts.append(
@@ -189,7 +234,7 @@ def anthropic_messages_to_openai(
tc = (
parts
if any(p["type"] == "image_url" for p in parts)
- else " ".join(p["text"] for p in parts)
+ else "\n".join(p["text"] for p in parts)
)
tool_results.append(
{
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index b77bcbe6c43..f12302975b9 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -3296,15 +3296,17 @@ class AnthropicToolResultBlock(BaseModel):
return "" if v is None else v
-# Block types the converter translates explicitly. Anything else (thinking / redacted_thinking, a
-# provider block a resumed session replays, or a future type) is accepted as an unknown block and
-# dropped by the converter, rather than 400-ing the whole request on strict validation.
+# Block types with typed models. Anything else (a search_result or document, a provider block a
+# resumed session replays, or a future type) is accepted as an unknown block, which the converter
+# renders if it can and otherwise drops, rather than 400-ing the whole request on strict validation.
_KNOWN_ANTHROPIC_BLOCK_TYPES = frozenset(
{"text", "image", "tool_use", "tool_result", "thinking", "redacted_thinking"}
)
# Thinking blocks are replayed only in assistant turns; the converter drops them
# from user content, so accepting them there would silently lose a user turn.
-_USER_ANTHROPIC_BLOCK_TYPES = frozenset({"text", "image", "tool_use", "tool_result"})
+_USER_ANTHROPIC_BLOCK_TYPES = frozenset(
+ {"text", "image", "tool_use", "tool_result", "search_result", "document"}
+)
class AnthropicUnknownBlock(BaseModel):
@@ -3417,6 +3419,20 @@ class AnthropicMessage(BaseModel):
# value would raise TypeError, escaping as a 500 instead of a clean 400.
if not isinstance(btype, str) or btype not in _USER_ANTHROPIC_BLOCK_TYPES:
raise ValueError(f"unsupported content block type {btype!r} in a user message")
+ # A PDF, url or file document cannot be read. Only inside a tool result, which
+ # clients resend with history, does it degrade to a note instead of a 400.
+ if btype == "document":
+ source = (
+ block.get("source")
+ if isinstance(block, dict)
+ else getattr(block, "source", None)
+ )
+ stype = source.get("type") if isinstance(source, dict) else None
+ if stype not in ("text", "content"):
+ raise ValueError(
+ f"unsupported document source type {stype!r}: only text and content "
+ "documents can be read"
+ )
return data
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index c1e2c2b548f..918cdaeb8a0 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -2040,7 +2040,11 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],
continue
part_type = part.get("type")
- # Match the native tool-result conversion: text and image blocks reach the wire.
+ if reference := anthropic_reference_block_text(part):
+ estimate_content.append({"type": "text", "text": reference})
+ continue
+
+ # Match the native tool-result conversion: only the blocks it renders reach the wire.
if part_type == "tool_result" and isinstance(part.get("content"), list):
part = dict(part)
tool_content = []
@@ -2049,6 +2053,8 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],
continue
if block.get("type") == "text":
tool_content.append(block)
+ elif reference := anthropic_reference_block_text(block):
+ tool_content.append({"type": "text", "text": reference})
elif block.get("type") == "image":
image_parts += 1
tool_content.append(_openai_llama_admission_compact_image_part(block))
@@ -3296,6 +3302,7 @@ from models.inference import (
)
from core.inference.anthropic_compat import (
anthropic_messages_to_openai,
+ anthropic_reference_block_text,
fold_tool_results_into_user,
anthropic_schema_client_tool_kind,
anthropic_tools_to_openai,
diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py
index c55cb649c0d..1133b8d56f6 100644
--- a/studio/backend/tests/test_anthropic_messages.py
+++ b/studio/backend/tests/test_anthropic_messages.py
@@ -28,6 +28,8 @@ from models.inference import (
AnthropicResponseToolUseBlock,
)
from core.inference.anthropic_compat import (
+ DOCUMENT_IMAGE_OMITTED,
+ DOCUMENT_OMITTED,
anthropic_messages_to_openai,
anthropic_schema_client_tool_kind,
anthropic_tools_to_openai,
@@ -1028,7 +1030,137 @@ class TestAnthropicMessagesToOpenAI:
}
]
result = anthropic_messages_to_openai(msgs)
- assert result[0]["content"] == "Line 1 Line 2"
+ assert result[0]["content"] == "Line 1\nLine 2"
+
+ def test_tool_result_search_results_keep_title_source_and_text(self):
+ msgs = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "tu_1",
+ "content": [
+ {
+ "type": "search_result",
+ "source": "https://docs.example.com/vault",
+ "title": "Vault",
+ "content": [
+ {"type": "text", "text": "The code is PURPLE-ELEPHANT-42."},
+ {"type": "text", "text": "Rotate it monthly."},
+ ],
+ "citations": {"enabled": True},
+ },
+ {
+ "type": "search_result",
+ "source": "kb://faq",
+ "title": "FAQ",
+ "content": [{"type": "text", "text": "Ask the admin."}],
+ },
+ ],
+ },
+ ],
+ }
+ ]
+ result = anthropic_messages_to_openai(msgs)
+ assert result == [
+ {
+ "role": "tool",
+ "tool_call_id": "tu_1",
+ "content": "Title: Vault\nSource: https://docs.example.com/vault\n"
+ "The code is PURPLE-ELEPHANT-42.\nRotate it monthly.\n"
+ "Title: FAQ\nSource: kb://faq\nAsk the admin.",
+ }
+ ]
+
+ @pytest.mark.parametrize(
+ "source, body",
+ [
+ ({"type": "text", "media_type": "text/plain", "data": "Plain body."}, "Plain body."),
+ (
+ {
+ "type": "content",
+ "content": [
+ {"type": "text", "text": "First chunk"},
+ {"type": "text", "text": "Second chunk"},
+ ],
+ },
+ "First chunk\nSecond chunk",
+ ),
+ ({"type": "content", "content": "Whole body."}, "Whole body."),
+ (
+ {
+ "type": "content",
+ "content": [
+ {"type": "text", "text": "Before"},
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": "image/png",
+ "data": "iVBOR",
+ },
+ },
+ {"type": "text", "text": "After"},
+ ],
+ },
+ f"Before\n{DOCUMENT_IMAGE_OMITTED}\nAfter",
+ ),
+ (
+ {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0="},
+ DOCUMENT_OMITTED,
+ ),
+ ({"type": "url", "url": "https://example.com/a.pdf"}, DOCUMENT_OMITTED),
+ ],
+ )
+ def test_tool_result_document_renders_its_readable_source(self, source, body):
+ document = {
+ "type": "document",
+ "source": source,
+ "title": "Handbook",
+ "context": "Internal",
+ }
+ msgs = [
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": "tu_1", "content": [document]}],
+ }
+ ]
+ result = anthropic_messages_to_openai(msgs)
+ assert result[0]["content"] == f"Title: Handbook\nContext: Internal\n{body}"
+
+ def test_top_level_search_result_and_document_reach_the_user_turn(self):
+ request = AnthropicMessagesRequest(
+ model = "x",
+ max_tokens = 16,
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "search_result",
+ "source": "kb://vault",
+ "title": "Vault",
+ "content": [{"type": "text", "text": "PURPLE-ELEPHANT-42"}],
+ },
+ {
+ "type": "document",
+ "source": {"type": "text", "media_type": "text/plain", "data": "Memo."},
+ },
+ {"type": "document", "source": {"type": "content", "content": "Notes."}},
+ {"type": "text", "text": "What is the code?"},
+ ],
+ }
+ ],
+ )
+ result = anthropic_messages_to_openai([m.model_dump() for m in request.messages])
+ assert result == [
+ {
+ "role": "user",
+ "content": "Title: Vault\nSource: kb://vault\nPURPLE-ELEPHANT-42\n"
+ "Memo.\nNotes.\nWhat is the code?",
+ }
+ ]
def test_image_base64_block_becomes_multimodal_part(self):
msgs = [
@@ -3889,11 +4021,41 @@ def test_user_unknown_block_rejected_not_silently_dropped():
model = "x",
max_tokens = 16,
messages = [
- {"role": "user", "content": [{"type": "document", "source": {}}]},
+ {"role": "user", "content": [{"type": "container_upload", "file_id": "f"}]},
],
)
+ "source",
+ [
+ {"type": "base64", "media_type": "application/pdf", "data": "JVBERi0="},
+ {"type": "url", "url": "https://example.com/a.pdf"},
+ {"type": "file", "file_id": "file_1"},
+ ],
+)
+def test_user_unreadable_document_rejected_outside_tool_results(source):
+ from pydantic import ValidationError
+
+ document = {"type": "document", "source": source}
+ with pytest.raises(ValidationError, match = "unsupported document source type"):
+ AnthropicMessagesRequest(
+ model = "x",
+ max_tokens = 16,
+ messages = [{"role": "user", "content": [{"type": "text", "text": "Read"}, document]}],
+ )
+ AnthropicMessagesRequest(
+ model = "x",
+ max_tokens = 16,
+ messages = [
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [document]}],
+ }
+ ],
+ )
+
+
def test_user_translatable_blocks_still_accepted():
# text / image / tool_result are translatable, so a real user message built from
# them must still pass; the unknown-block guard only trips on other types.
diff --git a/studio/backend/tests/test_anthropic_native_tool_images.py b/studio/backend/tests/test_anthropic_native_tool_images.py
index 6802769028b..0d84d89fc48 100644
--- a/studio/backend/tests/test_anthropic_native_tool_images.py
+++ b/studio/backend/tests/test_anthropic_native_tool_images.py
@@ -168,7 +168,7 @@ def test_native_image_http_generation_and_count(monkeypatch, vision):
assert url.startswith("data:image/png;base64,")
assert Image.open(BytesIO(base64.b64decode(url.split(",", 1)[1]))).size == (2, 2)
else:
- assert sent == counted_content == f"capture {TOOL_RESULT_IMAGE_OMITTED}"
+ assert sent == counted_content == f"capture\n{TOOL_RESULT_IMAGE_OMITTED}"
def test_text_only_tool_image_keeps_the_server_tool_permission_gate(monkeypatch):
diff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
index d409e4000a1..338303fd3ab 100644
--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
@@ -553,8 +553,8 @@ class TestAToolResultScreenshotIsNotPricedByItsBase64:
class TestEveryBlockTheTranslationDropsIsPricedTheSameWay:
"""`tool_result` content is an untyped list, so an image is only one of the block types
- that reach it. A document, a search result and a nested `tool_result` are dropped by
- the same translation filter, and each was charged its base64 as prompt text -- the
+ that reach it. A PDF document, a malformed search result and a nested `tool_result` send
+ at most a short note, and each was charged its base64 as prompt text -- the
whole of a 32768-token cache for a request that sends a couple of hundred characters.
"""
@@ -641,6 +641,41 @@ class TestEveryBlockTheTranslationDropsIsPricedTheSameWay:
estimate_messages
), f"{where}: the text beside it IS sent, so dropping it under-reserves"
+ def test_search_result_and_document_text_is_charged_where_it_is_sent(self):
+ search_result = {
+ "type": "search_result",
+ "source": "kb://vault",
+ "title": "Vault",
+ "content": [{"type": "text", "text": "PURPLE-ELEPHANT-42 " * 200}],
+ }
+ document = {
+ "type": "document",
+ "source": {"type": "text", "media_type": "text/plain", "data": "MEMO-BODY " * 200},
+ "title": "Memo",
+ }
+ for block, header in (
+ (search_result, "Title: Vault\nSource: kb://vault\n"),
+ (document, "Title: Memo\n"),
+ ):
+ for payload in (
+ self._request(block, text_first = True),
+ AnthropicMessagesRequest(
+ model = "default",
+ max_tokens = 128,
+ messages = [{"role": "user", "content": [block]}],
+ ),
+ ):
+ sent = anthropic_messages_to_openai([m.model_dump() for m in payload.messages])
+ rendered = sent[-1]["content"].rsplit("the tool answered\n", 1)[-1]
+ estimate_messages, image_parts = _openai_llama_admission_messages_for_estimate(
+ payload.messages
+ )
+ assert rendered.startswith(header)
+ assert rendered in str(estimate_messages).replace("\\n", "\n")
+ assert image_parts == 0
+ cost = _openai_llama_admission_tokens(payload, budget = 1_000_000, capacity = 4)
+ assert cost > len(rendered) // 8
+
def test_a_tool_result_the_translation_does_forward_is_still_charged(self):
"""The other side of the boundary: string `tool_result` content is forwarded
verbatim, base64-looking text included, so it keeps costing what its length costs