Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: request stream usage from llama.cpp connections so the context bar fills (#12688)

Jay Zhoufb889c2aa82d
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index f08c8aa40e9..5326d3b367c 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -42,11 +42,11 @@ _TEMPLATE_APPLYING_PROVIDERS = frozenset({"vllm", "llama_cpp", "ollama", "custom
# The subset documenting "continue_final_message" + "add_generation_prompt" on /v1/chat/completions.
_CONTINUATION_FLAG_PROVIDERS = frozenset({"vllm", "llama_cpp"})
-# The subset documenting stream_options.include_usage. An OAI-compatible stream omits usage without it, and these
-# providers report no llama.cpp timings either, so the monitor has no token count to derive a speed from. Same caution
-# as the flag above: "custom" is any user-supplied base_url and a strict endpoint 400s on an unknown field. "openai"
-# is absent because it routes to /v1/responses, which reports usage on its own.
-_USAGE_STREAM_OPTION_PROVIDERS = frozenset({"vllm", "openrouter", "kimi", "lemonade"})
+# The subset documenting stream_options.include_usage. An OAI-compatible stream omits usage without it, leaving the
+# chat context bar without prompt_tokens and, where no llama.cpp timings arrive, the monitor without a speed. Same
+# caution as the flag above: "custom" is any user-supplied base_url and a strict endpoint 400s on an unknown field.
+# "openai" is absent because it routes to /v1/responses, which reports usage on its own.
+_USAGE_STREAM_OPTION_PROVIDERS = frozenset({"vllm", "llama_cpp", "openrouter", "kimi", "lemonade"})
# llama-server reads repeat_penalty, not repetition_penalty (as routes/inference does).
_REPETITION_PENALTY_BODY_KEY = {"llama_cpp": "repeat_penalty"}
diff --git a/studio/backend/tests/test_external_provider_stream_usage_opt_in.py b/studio/backend/tests/test_external_provider_stream_usage_opt_in.py
index ab09e02d7d4..6a7b1c7e758 100644
--- a/studio/backend/tests/test_external_provider_stream_usage_opt_in.py
+++ b/studio/backend/tests/test_external_provider_stream_usage_opt_in.py
@@ -92,6 +92,13 @@ def test_only_an_opted_in_caller_receives_the_usage_chunk():
assert any(c.get("choices") for c in plain)
+def test_a_llama_cpp_connection_receives_the_usage_chunk():
+ # The chat context bar reads prompt_tokens from this chunk, not from llama-server timings.
+ opted_in, body = _proxy("llama_cpp", stream_options = {"include_usage": True})
+ assert body["stream_options"] == {"include_usage": True}
+ assert _usage_chunks(opted_in) == [_USAGE]
+
+
def test_the_opt_in_does_not_reach_a_custom_endpoint():
# "custom" is never asked for usage upstream: a strict endpoint 400s on the field.
_, opted_in = _proxy("custom", stream_options = {"include_usage": True})
diff --git a/studio/backend/tests/test_external_provider_usage_chunk.py b/studio/backend/tests/test_external_provider_usage_chunk.py
index 84bf77e693a..e8075b811c6 100644
--- a/studio/backend/tests/test_external_provider_usage_chunk.py
+++ b/studio/backend/tests/test_external_provider_usage_chunk.py
@@ -712,6 +712,7 @@ def test_other_providers_do_not_get_the_continuation_flags(monkeypatch, provider
("vllm", True),
("openrouter", True),
("kimi", True),
+ ("llama_cpp", True),
# Any user-supplied base_url: a strict endpoint 400s on an unknown field.
("custom", False),
("ollama", False),