Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: request stream usage from llama.cpp connections so the context bar fills (#12688)
Jay Zhoufb889c2aa82d
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.pyindex f08c8aa40e9..5326d3b367c 100644--- a/studio/backend/core/inference/external_provider.py+++ b/studio/backend/core/inference/external_provider.py@@ -42,11 +42,11 @@ _TEMPLATE_APPLYING_PROVIDERS = frozenset({"vllm", "llama_cpp", "ollama", "custom# The subset documenting "continue_final_message" + "add_generation_prompt" on /v1/chat/completions._CONTINUATION_FLAG_PROVIDERS = frozenset({"vllm", "llama_cpp"})-# The subset documenting stream_options.include_usage. An OAI-compatible stream omits usage without it, and these-# providers report no llama.cpp timings either, so the monitor has no token count to derive a speed from. Same caution-# as the flag above: "custom" is any user-supplied base_url and a strict endpoint 400s on an unknown field. "openai"-# is absent because it routes to /v1/responses, which reports usage on its own.-_USAGE_STREAM_OPTION_PROVIDERS = frozenset({"vllm", "openrouter", "kimi", "lemonade"})+# The subset documenting stream_options.include_usage. An OAI-compatible stream omits usage without it, leaving the+# chat context bar without prompt_tokens and, where no llama.cpp timings arrive, the monitor without a speed. Same+# caution as the flag above: "custom" is any user-supplied base_url and a strict endpoint 400s on an unknown field.+# "openai" is absent because it routes to /v1/responses, which reports usage on its own.+_USAGE_STREAM_OPTION_PROVIDERS = frozenset({"vllm", "llama_cpp", "openrouter", "kimi", "lemonade"})# llama-server reads repeat_penalty, not repetition_penalty (as routes/inference does)._REPETITION_PENALTY_BODY_KEY = {"llama_cpp": "repeat_penalty"}diff --git a/studio/backend/tests/test_external_provider_stream_usage_opt_in.py b/studio/backend/tests/test_external_provider_stream_usage_opt_in.pyindex ab09e02d7d4..6a7b1c7e758 100644--- a/studio/backend/tests/test_external_provider_stream_usage_opt_in.py+++ b/studio/backend/tests/test_external_provider_stream_usage_opt_in.py@@ -92,6 +92,13 @@ def test_only_an_opted_in_caller_receives_the_usage_chunk():assert any(c.get("choices") for c in plain)+def test_a_llama_cpp_connection_receives_the_usage_chunk():+ # The chat context bar reads prompt_tokens from this chunk, not from llama-server timings.+ opted_in, body = _proxy("llama_cpp", stream_options = {"include_usage": True})+ assert body["stream_options"] == {"include_usage": True}+ assert _usage_chunks(opted_in) == [_USAGE]++def test_the_opt_in_does_not_reach_a_custom_endpoint():# "custom" is never asked for usage upstream: a strict endpoint 400s on the field._, opted_in = _proxy("custom", stream_options = {"include_usage": True})diff --git a/studio/backend/tests/test_external_provider_usage_chunk.py b/studio/backend/tests/test_external_provider_usage_chunk.pyindex 84bf77e693a..e8075b811c6 100644--- a/studio/backend/tests/test_external_provider_usage_chunk.py+++ b/studio/backend/tests/test_external_provider_usage_chunk.py@@ -712,6 +712,7 @@ def test_other_providers_do_not_get_the_continuation_flags(monkeypatch, provider("vllm", True),("openrouter", True),("kimi", True),+ ("llama_cpp", True),# Any user-supplied base_url: a strict endpoint 400s on an unknown field.("custom", False),("ollama", False),