Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop pricing an Anthropic image by its base64 length in KV admission (#10669)
Nilaydb3e48a2ae2c
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 3db0f20ad46..557a404b453 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1947,6 +1947,29 @@ def _openai_llama_admission_image_tokens(llama_backend) -> int:return cap + _OPENAI_LLAMA_ADMISSION_IMAGE_WRAPPER_TOKENS+_ADMISSION_IMAGE_PART_TYPES = ("image_url", "image")+++def _openai_llama_admission_compact_image_part(part: dict) -> dict:+ """One image part with its transport bytes replaced by a marker.++ Keeps the wrapper the part really has, since that little JSON is prompt text the+ request does send; only the base64 goes.+ """+ if part.get("type") == "image":+ source = part.get("source")+ compact_source = {"type": "base64", "data": "[image]"}+ if isinstance(source, dict) and source.get("media_type") is not None:+ compact_source["media_type"] = source["media_type"]+ return {"type": "image", "source": compact_source}++ image_url = part.get("image_url")+ compact_image_url = {"url": "[image]"}+ if isinstance(image_url, dict) and image_url.get("detail") is not None:+ compact_image_url["detail"] = image_url["detail"]+ return {"type": "image_url", "image_url": compact_image_url}++def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict], int]:"""Remove image bytes before estimating the textual part of a prompt.@@ -1969,21 +1992,31 @@ def _openai_llama_admission_messages_for_estimate(messages) -> tuple[list[dict],if isinstance(content, list):estimate_content = []for part in content:- if not isinstance(part, dict) or part.get("type") != "image_url":+ if not isinstance(part, dict):+ estimate_content.append(part)+ continue++ part_type = part.get("type")+ # The same filter anthropic_messages_to_openai applies, so the estimate+ # charges what that sends. The content list is untyped, so a screenshot an+ # agent's tool returned, a document and a future block type all arrive+ # here, and none of them reach the wire to earn an allowance.+ if part_type == "tool_result" and isinstance(part.get("content"), list):+ part = dict(part)+ part["content"] = [+ block+ for block in part["content"]+ if isinstance(block, dict) and block.get("type") == "text"+ ]+ estimate_content.append(part)+ continue++ if part_type not in _ADMISSION_IMAGE_PART_TYPES:estimate_content.append(part)continueimage_parts += 1- image_url = part.get("image_url")- compact_image_url = {"url": "[image]"}- if isinstance(image_url, dict) and image_url.get("detail") is not None:- compact_image_url["detail"] = image_url["detail"]- estimate_content.append(- {- "type": "image_url",- "image_url": compact_image_url,- }- )+ estimate_content.append(_openai_llama_admission_compact_image_part(part))estimate_message["content"] = estimate_contentestimate_messages.append(estimate_message)return estimate_messages, image_partsdiff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.pyindex 94ffab4af43..903f5772306 100644--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py@@ -16,10 +16,13 @@ estimate for a lease that runs up to 25 growing rounds.import base64+from models.inference import AnthropicMessagesRequestfrom routes.inference import (_OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,+ _openai_llama_admission_messages_for_estimate,_openai_llama_admission_tokens,)+from core.inference.anthropic_compat import anthropic_messages_to_openaifrom core.inference.llama_admission import LlamaAdmissionConfig, LlamaAdmissionQueuefrom routes.inference import _openai_llama_admission_budgetimport asyncio@@ -362,6 +365,291 @@ class TestMediaIsCharged:assert cost > 2000, f"{field} was charged {cost}, i.e. nothing for the media"+class TestAnAnthropicImageIsChargedLikeAnyOtherImage:+ """/v1/messages reserves from the RAW Anthropic request, so its own image block has to+ be compacted too. #9842 fixed this for /v1/chat/completions and left this surface+ pricing a screenshot at its base64 length, which clamps the reservation to the whole+ cache and makes the shared queue serve that one request alone.+ """++ def _request(self, data: str):+ return AnthropicMessagesRequest(+ model = "default",+ max_tokens = 128,+ messages = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "what is this?"},+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": data,+ },+ },+ ],+ }+ ],+ )++ def test_a_big_anthropic_image_costs_what_a_tiny_one_costs(self):+ # Clear of the clamp, as the image_url cases above are, so the estimator is what+ # is being compared rather than `min(budget, ...)`.+ big = _openai_llama_admission_tokens(+ self._request(_image_b64(1024)), budget = 1_000_000, capacity = 4+ )+ tiny = _openai_llama_admission_tokens(self._request("AAAA"), budget = 1_000_000, capacity = 4)+ assert abs(big - tiny) <= _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS, (+ f"a 1 MiB Anthropic image was charged {big} against {tiny} for a 4-char one: "+ "the base64 transport is being priced as prompt text"+ )+ assert (+ big >= _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS+ ), "image bytes must be bounded but still charged"++ def test_a_screenshot_does_not_reserve_the_whole_cache(self):+ budget = 32768+ cost = _openai_llama_admission_tokens(+ self._request(_image_b64(150)), budget = budget, capacity = 4+ )+ assert cost < budget, (+ f"a 150 KiB screenshot was charged {cost} against a {budget}-token cache, so "+ "the queue admits it alone and every other chat waits"+ )++ def test_the_estimate_does_not_carry_the_base64(self):+ data = _image_b64(64)+ estimate_messages, image_parts = _openai_llama_admission_messages_for_estimate(+ self._request(data).messages+ )+ assert image_parts == 1, "the bounded per-image allowance is keyed on this count"+ assert data not in str(estimate_messages)+++class TestAToolResultScreenshotIsNotPricedByItsBase64:+ """The shape an agent actually sends: the image arrives nested in a `tool_result`,+ not as a top-level block. A 150 KiB screenshot returned by a tool was charged 51,433+ tokens against a 32768-token cache -- the whole of it -- so the chat that took the+ screenshot then ran alone.+ """++ def _request(self, data: str):+ return AnthropicMessagesRequest(+ model = "default",+ max_tokens = 128,+ messages = [+ {"role": "user", "content": [{"type": "text", "text": "take a screenshot"}]},+ {+ "role": "assistant",+ "content": [+ {"type": "tool_use", "id": "toolu_01", "name": "screenshot", "input": {}}+ ],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "toolu_01",+ "content": [+ {"type": "text", "text": "screenshot taken"},+ {+ "type": "image",+ "source": {+ "type": "base64",+ "media_type": "image/png",+ "data": data,+ },+ },+ ],+ }+ ],+ },+ ],+ )++ def test_a_screenshot_a_tool_returned_does_not_reserve_the_whole_cache(self):+ budget = 32768+ cost = _openai_llama_admission_tokens(+ self._request(_image_b64(150)), budget = budget, capacity = 4+ )+ assert cost < budget, (+ f"a 150 KiB tool-result screenshot was charged {cost} against a {budget}-token "+ "cache, so the agent that took it runs alone"+ )++ def test_a_big_tool_result_screenshot_costs_what_a_tiny_one_costs(self):+ big = _openai_llama_admission_tokens(+ self._request(_image_b64(1024)), budget = 1_000_000, capacity = 4+ )+ tiny = _openai_llama_admission_tokens(self._request("AAAA"), budget = 1_000_000, capacity = 4)+ assert abs(big - tiny) <= _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS, (+ f"a 1 MiB tool-result image was charged {big} against {tiny} for a 4-char one: "+ "the base64 transport is being priced as prompt text"+ )++ def test_the_charge_matches_what_the_translation_actually_sends(self):+ """The two halves have to agree, so this fails on whichever side moves first.++ `anthropic_messages_to_openai` keeps only the text blocks of a list+ `tool_result`, so the nested image never reaches llama-server and earns no mtmd+ allowance. Start forwarding it and this fails, which is the reminder that+ admission has to start charging for it.+ """+ data = _image_b64(64)+ payload = self._request(data)++ estimate_messages, image_parts = _openai_llama_admission_messages_for_estimate(+ payload.messages+ )+ assert data not in str(estimate_messages), "the base64 must not be priced as text"++ sent = anthropic_messages_to_openai(+ [message.model_dump() for message in payload.messages], None+ )+ forwarded = data in str(sent)+ assert image_parts == (1 if forwarded else 0), (+ "admission charges a bounded image allowance exactly when the translation "+ f"sends the image (forwarded={forwarded}, image_parts={image_parts})"+ )+++class TestEveryBlockTheTranslationDropsIsPricedTheSameWay:+ """`tool_result` content is an untyped list, so an image is only one of the block types+ that reach it. A document, a search result and a nested `tool_result` are dropped by+ the same translation filter, and each was charged its base64 as prompt text -- the+ whole of a 32768-token cache for a request that sends a couple of hundred characters.+ """++ def _blocks(self, data: str):+ return {+ "document": {+ "type": "document",+ "source": {"type": "base64", "media_type": "application/pdf", "data": data},+ },+ "search_result": {"type": "search_result", "source": {"data": data}},+ "nested tool_result": {+ "type": "tool_result",+ "tool_use_id": "toolu_02",+ "content": [+ {+ "type": "image",+ "source": {"type": "base64", "media_type": "image/png", "data": data},+ }+ ],+ },+ }++ def _request(self, block, *, text_first: bool):+ text = {"type": "text", "text": "the tool answered"}+ content = [text, block] if text_first else [block, text]+ return AnthropicMessagesRequest(+ model = "default",+ max_tokens = 128,+ messages = [+ {"role": "user", "content": [{"type": "text", "text": "use the tool"}]},+ {+ "role": "assistant",+ "content": [+ {"type": "tool_use", "id": "toolu_01", "name": "lookup", "input": {}}+ ],+ },+ {+ "role": "user",+ "content": [+ {"type": "tool_result", "tool_use_id": "toolu_01", "content": content}+ ],+ },+ ],+ )++ def test_none_of_them_reserve_the_whole_cache(self):+ budget = 32768+ data = _image_b64(150)+ # Both orders: a filter that stops at the first block would pass one of them.+ for text_first in (True, False):+ for name, block in self._blocks(data).items():+ payload = self._request(block, text_first = text_first)+ cost = _openai_llama_admission_tokens(payload, budget = budget, capacity = 4)+ assert cost < budget, (+ f"a 150 KiB {name} block (text_first={text_first}) was charged {cost} "+ f"against a {budget}-token cache, so that agent runs alone"+ )++ def test_the_charge_matches_what_the_translation_actually_sends(self):+ """Tied to the translation, not to a number, so it fails on whichever side moves.++ The text beside these blocks IS sent, which is what stops a filter that simply+ drops the whole `tool_result` from passing.+ """+ data = _image_b64(64)+ for text_first in (True, False):+ for name, block in self._blocks(data).items():+ where = f"{name} (text_first={text_first})"+ payload = self._request(block, text_first = text_first)+ estimate_messages, image_parts = _openai_llama_admission_messages_for_estimate(+ payload.messages+ )+ sent = anthropic_messages_to_openai(+ [message.model_dump() for message in payload.messages], None+ )+ assert data not in str(sent), f"{where}: the translation now forwards this block"+ assert data not in str(+ estimate_messages+ ), f"{where}: the transport is being priced as prompt text"+ assert (+ image_parts == 0+ ), f"{where}: charged {image_parts} image allowances for a dropped block"+ assert "the tool answered" in str(+ estimate_messages+ ), f"{where}: the text beside it IS sent, so dropping it under-reserves"++ def test_a_tool_result_the_translation_does_forward_is_still_charged(self):+ """The other side of the boundary: string `tool_result` content is forwarded+ verbatim, base64-looking text included, so it keeps costing what its length costs+ and the filter cannot pay for itself by dropping what IS sent.+ """+ data = _image_b64(150)+ payload = AnthropicMessagesRequest(+ model = "default",+ max_tokens = 128,+ messages = [+ {"role": "user", "content": [{"type": "text", "text": "use the tool"}]},+ {+ "role": "assistant",+ "content": [+ {"type": "tool_use", "id": "toolu_01", "name": "lookup", "input": {}}+ ],+ },+ {+ "role": "user",+ "content": [+ {"type": "tool_result", "tool_use_id": "toolu_01", "content": data}+ ],+ },+ ],+ )+ sent = anthropic_messages_to_openai(+ [message.model_dump() for message in payload.messages], None+ )+ assert data in str(sent), "the translation stopped forwarding string tool_result content"++ estimate_messages, image_parts = _openai_llama_admission_messages_for_estimate(+ payload.messages+ )+ assert data in str(+ estimate_messages+ ), "content that IS sent was dropped from the estimate, which under-reserves"+ assert image_parts == 0, "a string tool result is prompt text, not an image"++ cost = _openai_llama_admission_tokens(payload, budget = 1_000_000, capacity = 4)+ assert (+ cost > len(data) // 8+ ), f"a {len(data)}-char forwarded tool result was charged only {cost}"++class TestTheToolLoopOpensAtAnEqualShare:"""#9392 reserved the WHOLE cache for any tool loop, making every tool chat run alone(any lit pill sets enable_tools). The loop now opens at an equal share and re-costs