Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: report prompt cache hits on the Messages and Responses APIs, and ask vLLM and SGLang for them (#13741)
Daniel Hanab0abb72e375
diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.pyindex b4124e49575..5d22ac70a48 100644--- a/studio/backend/core/inference/anthropic_compat.py+++ b/studio/backend/core/inference/anthropic_compat.py@@ -602,15 +602,33 @@ def build_anthropic_sse_event(event_type: str, data: dict) -> str:return f"event: {event_type}\ndata: {json.dumps(data)}\n\n"+def cached_prompt_tokens(usage: Optional[dict]) -> Optional[int]:+ """Prompt tokens the server reused from its cache (llama-server ``prompt_tokens_details``), or None."""+ details = (usage or {}).get("prompt_tokens_details")+ cached = details.get("cached_tokens") if isinstance(details, dict) else None+ if isinstance(cached, bool) or not isinstance(cached, (int, float)):+ return None+ return max(0, int(cached))+++def anthropic_input_split(usage: Optional[dict]) -> tuple[int, int]:+ """(input_tokens, cache_read_input_tokens): Anthropic counts cache reads apart from input, so the two+ still sum to the prompt. Nothing is ever reported as cache creation."""+ usage = usage or {}+ prompt = int(usage.get("prompt_tokens") or 0)+ cached = min(prompt, cached_prompt_tokens(usage) or 0)+ return prompt - cached, cached++def _message_delta_usage(usage: Optional[dict]) -> dict:- """Usage block for a message_delta event (cumulative token counts). Cache- fields are always 0 — no prompt caching backend. ``usage`` may be None when a+ """Usage block for a message_delta event (cumulative token counts). ``usage`` may be None when ametadata event carried usage=None (e.g. only finish_reason set)."""usage = usage or {}+ input_tokens, cache_read = anthropic_input_split(usage)return {- "input_tokens": usage.get("prompt_tokens", 0),+ "input_tokens": input_tokens,"cache_creation_input_tokens": 0,- "cache_read_input_tokens": 0,+ "cache_read_input_tokens": cache_read,"output_tokens": usage.get("completion_tokens", 0),}diff --git a/studio/backend/core/inference/engine_adapters.py b/studio/backend/core/inference/engine_adapters.pyindex 6c15360c99b..fdc2ef054d6 100644--- a/studio/backend/core/inference/engine_adapters.py+++ b/studio/backend/core/inference/engine_adapters.py@@ -260,6 +260,8 @@ class EngineAdapter:else []),*(["--trust-remote-code"] if trust_remote_code else []),+ # Prefix caching is on by default in both; this only reports its hits in usage.+ "--enable-prompt-tokens-details" if self.name == "vllm" else "--enable-cache-report",*self.extra_args,"--host","127.0.0.1",diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 178850992de..e0aaf59818b 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -3954,6 +3954,8 @@ class ResponsesUsage(BaseModel):input_tokens: int = 0output_tokens: int = 0total_tokens: int = 0+ # {"cached_tokens": n}: prompt tokens the server reused from its cache, when it said.+ input_tokens_details: Optional[dict[str, int]] = Noneclass ResponsesResponse(BaseModel):diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex fb7dd4875db..a53ca33cddd 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -3827,6 +3827,8 @@ from models.inference import ()from core.inference.anthropic_compat import (TOOL_RESULT_IMAGE_OMITTED,+ anthropic_input_split,+ cached_prompt_tokens,anthropic_messages_to_openai,anthropic_reference_block_text,fold_tool_results_into_user,@@ -39089,6 +39091,7 @@ async def _responses_non_streaming(usage_data = body.get("usage", {})input_tokens = usage_data.get("prompt_tokens", 0)output_tokens = usage_data.get("completion_tokens", 0)+ cached_input = cached_prompt_tokens(usage_data)resp_id = f"resp_{uuid.uuid4().hex[:12]}"@@ -39132,6 +39135,9 @@ async def _responses_non_streaming(input_tokens = input_tokens,output_tokens = output_tokens,total_tokens = input_tokens + output_tokens,+ input_tokens_details = (+ None if cached_input is None else {"cached_tokens": cached_input}+ ),),temperature = payload.temperature,top_p = payload.top_p,@@ -39326,6 +39332,7 @@ async def _responses_stream(full_reasoning = ""input_tokens = 0output_tokens = 0+ cached_input: Optional[int] = None# From the chat chunks; applied once before finish, so chunk order does not matter.stream_finish_reason: Optional[str] = Noneextractor = _ResponsesReasoningExtractor(@@ -39444,11 +39451,13 @@ async def _responses_stream(return output_indexdef _apply_usage(u, timings = None) -> None:- nonlocal input_tokens, output_tokens+ nonlocal input_tokens, output_tokens, cached_input# No early return on a falsy usage: the final chunk can carry timings alone.if isinstance(u, dict):input_tokens = u.get("prompt_tokens", input_tokens)output_tokens = u.get("completion_tokens", output_tokens)+ cached = cached_prompt_tokens(u)+ cached_input = cached_input if cached is None else cached_monitor_usage(monitor_id, u, llama_backend.context_length, timings = timings)def _ensure_reasoning_open() -> list[str]:@@ -40214,6 +40223,11 @@ async def _responses_stream("input_tokens": input_tokens,"output_tokens": output_tokens,"total_tokens": total_tokens,+ **(+ {}+ if cached_input is None+ else {"input_tokens_details": {"cached_tokens": cached_input}}+ ),},},}@@ -43266,6 +43280,7 @@ def _anthropic_message_json_response() -> Response:"""Assemble the terminal Anthropic non-streaming JSON response shared by thetool / plain / passthrough paths."""+ input_tokens, cache_read = anthropic_input_split(usage)return _model_json_response(AnthropicMessagesResponse(id = message_id,@@ -43273,7 +43288,8 @@ def _anthropic_message_json_response(content = content_blocks,stop_reason = stop_reason,usage = AnthropicUsage(- input_tokens = usage.get("prompt_tokens", 0),+ input_tokens = input_tokens,+ cache_read_input_tokens = cache_read,output_tokens = usage.get("completion_tokens", 0),),)diff --git a/studio/backend/tests/test_cached_token_usage.py b/studio/backend/tests/test_cached_token_usage.pynew file mode 100644index 00000000000..ccc20a603e1--- /dev/null+++ b/studio/backend/tests/test_cached_token_usage.py@@ -0,0 +1,67 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Prompt cache hits reach the clients that read them, on the Messages and Responses APIs."""++from __future__ import annotations++import json++from core.inference import anthropic_compat+from core.inference.anthropic_compat import anthropic_input_split, cached_prompt_tokens+from core.inference.engine_adapters import launch_arguments+from models.inference import ResponsesUsage++LLAMA_USAGE = {+ "prompt_tokens": 3568,+ "completion_tokens": 9,+ "prompt_tokens_details": {"cached_tokens": 3254},+}+++def test_cache_reads_are_reported_apart_from_input_and_still_sum_to_the_prompt():+ assert anthropic_input_split(LLAMA_USAGE) == (314, 3254)+ delta = anthropic_compat._message_delta_usage(LLAMA_USAGE)+ assert delta["cache_read_input_tokens"] == 3254+ assert delta["input_tokens"] + delta["cache_read_input_tokens"] == 3568+ assert delta["cache_creation_input_tokens"] == 0+++def test_a_server_that_says_nothing_about_its_cache_reports_as_before():+ usage = {"prompt_tokens": 100, "completion_tokens": 5}++ assert cached_prompt_tokens(usage) is None+ assert anthropic_compat._message_delta_usage(usage) == {+ "input_tokens": 100,+ "cache_creation_input_tokens": 0,+ "cache_read_input_tokens": 0,+ "output_tokens": 5,+ }+ assert anthropic_compat._message_delta_usage(None)["input_tokens"] == 0+++def test_a_cached_count_above_the_prompt_is_clamped():+ usage = {"prompt_tokens": 10, "prompt_tokens_details": {"cached_tokens": 50}}++ assert anthropic_input_split(usage) == (0, 10)+++def test_responses_usage_carries_cached_tokens_only_when_known():+ known = json.loads(+ ResponsesUsage(+ input_tokens = 3568, input_tokens_details = {"cached_tokens": 3254}+ ).model_dump_json()+ )+ assert known["input_tokens_details"] == {"cached_tokens": 3254}+ assert (+ json.loads(ResponsesUsage(input_tokens = 1).model_dump_json())["input_tokens_details"] is None+ )+++def test_managed_engines_are_asked_to_report_their_cache_hits():+ for name, flag in (+ ("vllm", "--enable-prompt-tokens-details"),+ ("sglang", "--enable-cache-report"),+ ):+ args = launch_arguments(name, "python", "org/model", 8000, "key", 4096)+ assert flag in args, namediff --git a/studio/backend/tests/test_managed_engine_launch_options.py b/studio/backend/tests/test_managed_engine_launch_options.pyindex f4286a1a912..e08684710bd 100644--- a/studio/backend/tests/test_managed_engine_launch_options.py+++ b/studio/backend/tests/test_managed_engine_launch_options.py@@ -39,6 +39,7 @@ _MAIN_ARGV = {"--tool-call-parser","hermes","--enable-auto-tool-choice",+ "--enable-prompt-tokens-details","--generation-config","vllm","--host",@@ -63,6 +64,7 @@ _MAIN_ARGV = {"1","--tool-call-parser","hermes",+ "--enable-cache-report","--attention-backend","triton","--sampling-backend",