Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: report prompt cache hits on the Messages and Responses APIs, and ask vLLM and SGLang for them (#13741)

Daniel Hanab0abb72e375
diff --git a/studio/backend/core/inference/anthropic_compat.py b/studio/backend/core/inference/anthropic_compat.py
index b4124e49575..5d22ac70a48 100644
--- a/studio/backend/core/inference/anthropic_compat.py
+++ b/studio/backend/core/inference/anthropic_compat.py
@@ -602,15 +602,33 @@ def build_anthropic_sse_event(event_type: str, data: dict) -> str:
return f"event: {event_type}\ndata: {json.dumps(data)}\n\n"
+def cached_prompt_tokens(usage: Optional[dict]) -> Optional[int]:
+ """Prompt tokens the server reused from its cache (llama-server ``prompt_tokens_details``), or None."""
+ details = (usage or {}).get("prompt_tokens_details")
+ cached = details.get("cached_tokens") if isinstance(details, dict) else None
+ if isinstance(cached, bool) or not isinstance(cached, (int, float)):
+ return None
+ return max(0, int(cached))
+
+
+def anthropic_input_split(usage: Optional[dict]) -> tuple[int, int]:
+ """(input_tokens, cache_read_input_tokens): Anthropic counts cache reads apart from input, so the two
+ still sum to the prompt. Nothing is ever reported as cache creation."""
+ usage = usage or {}
+ prompt = int(usage.get("prompt_tokens") or 0)
+ cached = min(prompt, cached_prompt_tokens(usage) or 0)
+ return prompt - cached, cached
+
+
def _message_delta_usage(usage: Optional[dict]) -> dict:
- """Usage block for a message_delta event (cumulative token counts). Cache
- fields are always 0 — no prompt caching backend. ``usage`` may be None when a
+ """Usage block for a message_delta event (cumulative token counts). ``usage`` may be None when a
metadata event carried usage=None (e.g. only finish_reason set)."""
usage = usage or {}
+ input_tokens, cache_read = anthropic_input_split(usage)
return {
- "input_tokens": usage.get("prompt_tokens", 0),
+ "input_tokens": input_tokens,
"cache_creation_input_tokens": 0,
- "cache_read_input_tokens": 0,
+ "cache_read_input_tokens": cache_read,
"output_tokens": usage.get("completion_tokens", 0),
}
diff --git a/studio/backend/core/inference/engine_adapters.py b/studio/backend/core/inference/engine_adapters.py
index 6c15360c99b..fdc2ef054d6 100644
--- a/studio/backend/core/inference/engine_adapters.py
+++ b/studio/backend/core/inference/engine_adapters.py
@@ -260,6 +260,8 @@ class EngineAdapter:
else []
),
*(["--trust-remote-code"] if trust_remote_code else []),
+ # Prefix caching is on by default in both; this only reports its hits in usage.
+ "--enable-prompt-tokens-details" if self.name == "vllm" else "--enable-cache-report",
*self.extra_args,
"--host",
"127.0.0.1",
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 178850992de..e0aaf59818b 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -3954,6 +3954,8 @@ class ResponsesUsage(BaseModel):
input_tokens: int = 0
output_tokens: int = 0
total_tokens: int = 0
+ # {"cached_tokens": n}: prompt tokens the server reused from its cache, when it said.
+ input_tokens_details: Optional[dict[str, int]] = None
class ResponsesResponse(BaseModel):
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index fb7dd4875db..a53ca33cddd 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -3827,6 +3827,8 @@ from models.inference import (
)
from core.inference.anthropic_compat import (
TOOL_RESULT_IMAGE_OMITTED,
+ anthropic_input_split,
+ cached_prompt_tokens,
anthropic_messages_to_openai,
anthropic_reference_block_text,
fold_tool_results_into_user,
@@ -39089,6 +39091,7 @@ async def _responses_non_streaming(
usage_data = body.get("usage", {})
input_tokens = usage_data.get("prompt_tokens", 0)
output_tokens = usage_data.get("completion_tokens", 0)
+ cached_input = cached_prompt_tokens(usage_data)
resp_id = f"resp_{uuid.uuid4().hex[:12]}"
@@ -39132,6 +39135,9 @@ async def _responses_non_streaming(
input_tokens = input_tokens,
output_tokens = output_tokens,
total_tokens = input_tokens + output_tokens,
+ input_tokens_details = (
+ None if cached_input is None else {"cached_tokens": cached_input}
+ ),
),
temperature = payload.temperature,
top_p = payload.top_p,
@@ -39326,6 +39332,7 @@ async def _responses_stream(
full_reasoning = ""
input_tokens = 0
output_tokens = 0
+ cached_input: Optional[int] = None
# From the chat chunks; applied once before finish, so chunk order does not matter.
stream_finish_reason: Optional[str] = None
extractor = _ResponsesReasoningExtractor(
@@ -39444,11 +39451,13 @@ async def _responses_stream(
return output_index
def _apply_usage(u, timings = None) -> None:
- nonlocal input_tokens, output_tokens
+ nonlocal input_tokens, output_tokens, cached_input
# No early return on a falsy usage: the final chunk can carry timings alone.
if isinstance(u, dict):
input_tokens = u.get("prompt_tokens", input_tokens)
output_tokens = u.get("completion_tokens", output_tokens)
+ cached = cached_prompt_tokens(u)
+ cached_input = cached_input if cached is None else cached
_monitor_usage(monitor_id, u, llama_backend.context_length, timings = timings)
def _ensure_reasoning_open() -> list[str]:
@@ -40214,6 +40223,11 @@ async def _responses_stream(
"input_tokens": input_tokens,
"output_tokens": output_tokens,
"total_tokens": total_tokens,
+ **(
+ {}
+ if cached_input is None
+ else {"input_tokens_details": {"cached_tokens": cached_input}}
+ ),
},
},
}
@@ -43266,6 +43280,7 @@ def _anthropic_message_json_response(
) -> Response:
"""Assemble the terminal Anthropic non-streaming JSON response shared by the
tool / plain / passthrough paths."""
+ input_tokens, cache_read = anthropic_input_split(usage)
return _model_json_response(
AnthropicMessagesResponse(
id = message_id,
@@ -43273,7 +43288,8 @@ def _anthropic_message_json_response(
content = content_blocks,
stop_reason = stop_reason,
usage = AnthropicUsage(
- input_tokens = usage.get("prompt_tokens", 0),
+ input_tokens = input_tokens,
+ cache_read_input_tokens = cache_read,
output_tokens = usage.get("completion_tokens", 0),
),
)
diff --git a/studio/backend/tests/test_cached_token_usage.py b/studio/backend/tests/test_cached_token_usage.py
new file mode 100644
index 00000000000..ccc20a603e1
--- /dev/null
+++ b/studio/backend/tests/test_cached_token_usage.py
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Prompt cache hits reach the clients that read them, on the Messages and Responses APIs."""
+
+from __future__ import annotations
+
+import json
+
+from core.inference import anthropic_compat
+from core.inference.anthropic_compat import anthropic_input_split, cached_prompt_tokens
+from core.inference.engine_adapters import launch_arguments
+from models.inference import ResponsesUsage
+
+LLAMA_USAGE = {
+ "prompt_tokens": 3568,
+ "completion_tokens": 9,
+ "prompt_tokens_details": {"cached_tokens": 3254},
+}
+
+
+def test_cache_reads_are_reported_apart_from_input_and_still_sum_to_the_prompt():
+ assert anthropic_input_split(LLAMA_USAGE) == (314, 3254)
+ delta = anthropic_compat._message_delta_usage(LLAMA_USAGE)
+ assert delta["cache_read_input_tokens"] == 3254
+ assert delta["input_tokens"] + delta["cache_read_input_tokens"] == 3568
+ assert delta["cache_creation_input_tokens"] == 0
+
+
+def test_a_server_that_says_nothing_about_its_cache_reports_as_before():
+ usage = {"prompt_tokens": 100, "completion_tokens": 5}
+
+ assert cached_prompt_tokens(usage) is None
+ assert anthropic_compat._message_delta_usage(usage) == {
+ "input_tokens": 100,
+ "cache_creation_input_tokens": 0,
+ "cache_read_input_tokens": 0,
+ "output_tokens": 5,
+ }
+ assert anthropic_compat._message_delta_usage(None)["input_tokens"] == 0
+
+
+def test_a_cached_count_above_the_prompt_is_clamped():
+ usage = {"prompt_tokens": 10, "prompt_tokens_details": {"cached_tokens": 50}}
+
+ assert anthropic_input_split(usage) == (0, 10)
+
+
+def test_responses_usage_carries_cached_tokens_only_when_known():
+ known = json.loads(
+ ResponsesUsage(
+ input_tokens = 3568, input_tokens_details = {"cached_tokens": 3254}
+ ).model_dump_json()
+ )
+ assert known["input_tokens_details"] == {"cached_tokens": 3254}
+ assert (
+ json.loads(ResponsesUsage(input_tokens = 1).model_dump_json())["input_tokens_details"] is None
+ )
+
+
+def test_managed_engines_are_asked_to_report_their_cache_hits():
+ for name, flag in (
+ ("vllm", "--enable-prompt-tokens-details"),
+ ("sglang", "--enable-cache-report"),
+ ):
+ args = launch_arguments(name, "python", "org/model", 8000, "key", 4096)
+ assert flag in args, name
diff --git a/studio/backend/tests/test_managed_engine_launch_options.py b/studio/backend/tests/test_managed_engine_launch_options.py
index f4286a1a912..e08684710bd 100644
--- a/studio/backend/tests/test_managed_engine_launch_options.py
+++ b/studio/backend/tests/test_managed_engine_launch_options.py
@@ -39,6 +39,7 @@ _MAIN_ARGV = {
"--tool-call-parser",
"hermes",
"--enable-auto-tool-choice",
+ "--enable-prompt-tokens-details",
"--generation-config",
"vllm",
"--host",
@@ -63,6 +64,7 @@ _MAIN_ARGV = {
"1",
"--tool-call-parser",
"hermes",
+ "--enable-cache-report",
"--attention-backend",
"triton",
"--sampling-backend",