Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Show token usage and cache stats on connected provider chats (#11717)
Nilay06be728d0410
diff --git a/studio/backend/tests/test_external_provider_stream_usage_opt_in.py b/studio/backend/tests/test_external_provider_stream_usage_opt_in.pynew file mode 100644index 00000000000..ab09e02d7d4--- /dev/null+++ b/studio/backend/tests/test_external_provider_stream_usage_opt_in.py@@ -0,0 +1,100 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import json+import sys+from pathlib import Path++from starlette.requests import Request++import routes.inference as ri+from models.inference import ChatCompletionRequest++sys.path.insert(0, str(Path(__file__).resolve().parent))+from test_external_provider_sampling_over_the_wire import _Handler, _run, _Server # noqa: E402+++_USAGE = {+ "prompt_tokens": 1200,+ "completion_tokens": 7,+ "total_tokens": 1207,+ "prompt_tokens_details": {"cached_tokens": 1024},+}+++class _UsageHandler(_Handler):+ def do_POST(self) -> None: # noqa: N802+ length = int(self.headers.get("Content-Length") or 0)+ self.server.recorded.append(json.loads(self.rfile.read(length) or b"{}")) # type: ignore[attr-defined]+ sse = (+ 'data: {"choices":[{"index":0,"delta":{"content":"ok"}}]}\n\n'+ 'data: {"choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n'+ f'data: {json.dumps({"choices": [], "usage": _USAGE})}\n\n'+ "data: [DONE]\n\n"+ ).encode()+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.send_header("Content-Length", str(len(sse)))+ self.end_headers()+ self.wfile.write(sse)+++def _proxy(provider_type: str, **payload_fields) -> tuple[list[dict], dict]:+ async def receive() -> dict:+ return {"type": "http.disconnect"}++ chunks: list[dict] = []+ with _Server(handler = _UsageHandler) as server:+ payload = ChatCompletionRequest(+ provider_type = provider_type,+ provider_base_url = server.base_url,+ messages = [{"role": "user", "content": "hi"}],+ model = "a-model",+ stream = True,+ **payload_fields,+ )+ request = Request(+ {+ "type": "http",+ "method": "POST",+ "path": "/v1/chat/completions",+ "headers": [(b"content-type", b"application/json")],+ "query_string": b"",+ },+ receive,+ )++ async def go() -> None:+ response = await ri._proxy_to_external_provider(payload, request)+ async for part in response.body_iterator:+ text = part.decode() if isinstance(part, bytes) else part+ for line in text.splitlines():+ body = line[len("data:") :].strip() if line.startswith("data:") else ""+ if body and body != "[DONE]":+ chunks.append(json.loads(body))++ _run(go)+ assert server.bodies, "the route never reached the provider"+ return chunks, server.bodies[-1]+++def _usage_chunks(chunks: list[dict]) -> list[dict]:+ return [c["usage"] for c in chunks if c.get("choices") == [] and c.get("usage")]+++def test_only_an_opted_in_caller_receives_the_usage_chunk():+ opted_in, _ = _proxy("vllm", stream_options = {"include_usage": True})+ plain, _ = _proxy("vllm")+ assert _usage_chunks(opted_in) == [_USAGE]+ assert _usage_chunks(plain) == []+ assert any(c.get("choices") for c in plain)+++def test_the_opt_in_does_not_reach_a_custom_endpoint():+ # "custom" is never asked for usage upstream: a strict endpoint 400s on the field.+ _, opted_in = _proxy("custom", stream_options = {"include_usage": True})+ _, plain = _proxy("custom")+ assert opted_in == plain+ assert "stream_options" not in opted_indiff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex 29db7597ad7..43b988d7c46 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -6227,6 +6227,7 @@ export function createOpenAIStreamAdapter(model: externalSelection.modelId,messages: outboundMessages,stream: true,+ stream_options: { include_usage: true },// Never forwarded upstream (the proxy sends an explicit field list); the trailing assistant// turn is what asks a provider to continue....(continuation ? { continue_final_message: true } : {}),diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex eb0a1787967..1708dafeca2 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -682,8 +682,8 @@ export interface OpenAIChatCompletionsRequest {/** Anthropic fast-mode toggle. Opus 4.6 / 4.7 only; dropped silently elsewhere. */fast_mode?: boolean | null;/** Opt into the OpenAI-standard trailing usage chunk on streams. The backend only emits it when- * `include_usage` is set; the local chat UI sends it so the context-usage bar and tok/s- * readout populate. */+ * `include_usage` is set; the chat UI sends it for local and connected-provider models so the+ * context-usage bar and tok/s readout populate. */stream_options?: { include_usage?: boolean } | null;}diff --git a/studio/frontend/tests/external-sampling-payload.test.ts b/studio/frontend/tests/external-sampling-payload.test.tsindex 0fca938fd85..47f55187f7e 100644--- a/studio/frontend/tests/external-sampling-payload.test.ts+++ b/studio/frontend/tests/external-sampling-payload.test.ts@@ -206,3 +206,16 @@ test("the panel and the request read the same capability flags", () => {),);});++// The proxy drops the usage chunk for a caller that did not opt in, which hid the usage bar.+test("the external body opts into the stream usage chunk", () => {+ const optIn = externalBodyLiteral().properties.find(+ (property) =>+ ts.isPropertyAssignment(property) &&+ property.name.getText() === "stream_options",+ );+ assert.ok(optIn && ts.isPropertyAssignment(optIn), "stream_options missing");+ assert.deepEqual(new Function(`return (${optIn.initializer.getText()});`)(), {+ include_usage: true,+ });+});