Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: price the context meter off a tool loop's final pass, not the whole turn's completions (#12889)
SlippyChimpMG06bbb7f90bf7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 810cf74458f..41777084028 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -36424,6 +36424,9 @@ class LlamaCppBackend:"completion_tokens": _tc,"total_tokens": _fp + _tc,}+ # Earlier passes' completions are already inside _fp, so the context is _fp + this pass only.+ if _accumulated_completion_tokens:+ _usage["context_tokens"] = _fp + int(_fu.get("completion_tokens") or 0)# Preserve KV-cache hit details (cached_tokens) so the tool path# reports them like the standard non-tool path does, not always 0.if _fu.get("prompt_tokens_details"):diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.pyindex 98395f07d45..44b441695e3 100644--- a/studio/backend/models/inference.py+++ b/studio/backend/models/inference.py@@ -19,6 +19,7 @@ from pydantic import (PrivateAttr,Tag,field_validator,+ model_serializer,model_validator,)@@ -3456,6 +3457,8 @@ class CompletionUsage(BaseModel):prompt_tokens: int = 0completion_tokens: int = 0total_tokens: int = 0+ # Studio tool loops: final pass's prompt + completion (total_tokens sums every pass's completion).+ context_tokens: Optional[int] = Noneprompt_tokens_details: Optional[dict] = Field(default_factory = lambda: {"cached_tokens": 0, "audio_tokens": 0})@@ -3468,6 +3471,14 @@ class CompletionUsage(BaseModel):})+ @model_serializer(mode = "wrap")+ def _omit_unset_context_tokens(self, handler):+ # Only tool loops set it; elsewhere the OpenAI usage object stays byte-identical.+ data = handler(self)+ if isinstance(data, dict) and data.get("context_tokens") is None:+ data.pop("context_tokens", None)+ return data+class ChatCompletion(BaseModel):"""Non-streaming chat completion response."""diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 51b8b312d63..9523381a068 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1474,6 +1474,7 @@ def _openai_stream_usage_chunk(completion_tokens = _completion_tokens,total_tokens = _total_tokens,prompt_tokens_details = _prompt_tokens_details(_usage.get("prompt_tokens_details")),+ context_tokens = _usage.get("context_tokens"),),timings = stream_timings,)diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.pyindex 0cf2745a680..567a76ede62 100644--- a/studio/backend/tests/test_llama_cpp_tool_loop.py+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py@@ -4521,6 +4521,21 @@ def test_metadata_event_preserves_prompt_tokens_details(monkeypatch):assert usage["completion_tokens"] == 4+def test_metadata_event_omits_context_tokens_on_a_single_pass(monkeypatch):+ stream = [+ _sse({"content": "hi"}),+ _usage_done({"prompt_tokens": 5, "completion_tokens": 2}),+ _done(),+ ]+ backend, _ = _backend_and_payloads(monkeypatch, [stream])++ events = _run_tool_loop(backend, [{"role": "user", "content": "hi"}], [_web_search_tool()])++ usage = [e for e in events if e.get("type") == "metadata"][-1]["usage"]+ assert usage["total_tokens"] == 7+ assert "context_tokens" not in usage++def test_metadata_event_omits_prompt_tokens_details_when_absent(monkeypatch):"""No KV-cache block from the server -> the key isn't fabricated, so theroute falls back to its 0-default instead of reading a bogus value."""@@ -4538,6 +4553,39 @@ def test_metadata_event_omits_prompt_tokens_details_when_absent(monkeypatch):assert "prompt_tokens_details" not in metadata[-1]["usage"]+def test_metadata_event_context_tokens_count_earlier_passes_once(monkeypatch):+ """A tool pass's completion is re-sent inside the next pass's prompt, so the+ context the turn leaves is the final prompt plus the final completion only;+ total_tokens keeps billing every pass's completion."""+ streams = [+ [+ _tool_call_sse("web_search", {"query": "cats"}, "call_1"),+ _usage_done({"prompt_tokens": 100, "completion_tokens": 30}, "tool_calls"),+ _done(),+ ],+ [+ _sse({"content": "Found cats."}),+ _usage_done({"prompt_tokens": 140, "completion_tokens": 10}),+ _done(),+ ],+ ]+ backend, payloads = _backend_and_payloads(monkeypatch, streams)+ _record_tool_calls(monkeypatch, "result")++ events = _run_tool_loop(+ backend,+ [{"role": "user", "content": "search cats"}],+ [_web_search_tool()],+ max_tool_iterations = 2,+ )++ assert len(payloads) == 2+ usage = [e for e in events if e.get("type") == "metadata"][-1]["usage"]+ assert usage["completion_tokens"] == 40+ assert usage["total_tokens"] == 180+ assert usage["context_tokens"] == 150++def test_gguf_rehearsal_name_split_before_args_is_not_leaked(monkeypatch):"""Finding 6: a rehearsal call whose name (``web_search``) and ``[ARGS]{...}``arrive in separate content deltas must hold the bare name in the buffer untildiff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex 075b4b3128e..0ffcb617831 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -3437,6 +3437,32 @@ class TestOpenAICompatibilityHelpers:assert usage["completion_tokens"] == 7assert usage["total_tokens"] == 7+ def test_stream_usage_chunk_forwards_tool_loop_context_tokens(self):+ payload = SimpleNamespace(stream_options = {"include_usage": True})+ tool_loop = {+ "prompt_tokens": 140,+ "completion_tokens": 40,+ "total_tokens": 180,+ "context_tokens": 150,+ }+ line = _openai_stream_usage_chunk(payload, "c", 1, "m", tool_loop, None)+ usage = json.loads(line.removeprefix("data: "))["usage"]+ assert usage["total_tokens"] == 180+ assert usage["context_tokens"] == 150++ single = {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}+ line = _openai_stream_usage_chunk(payload, "c", 1, "m", single, None)+ assert "context_tokens" not in json.loads(line.removeprefix("data: "))["usage"]++ def test_non_streaming_usage_omits_unset_context_tokens(self):+ from models.inference import CompletionUsage++ assert "context_tokens" not in json.loads(CompletionUsage(total_tokens = 5).model_dump_json())+ assert "context_tokens" not in CompletionUsage(total_tokens = 5).model_dump()+ assert (+ json.loads(CompletionUsage(context_tokens = 4).model_dump_json())["context_tokens"] == 4+ )+def test_completion_stream_monitor_reads_usage_before_client_strip(self, monkeypatch):import routes.inference as inf_moddiff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.tsindex 7c223695267..4128888811d 100644--- a/studio/frontend/src/features/chat/api/chat-adapter.ts+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts@@ -455,6 +455,8 @@ interface ServerUsage {prompt_tokens: number;completion_tokens: number;total_tokens: number;+ // Studio tool loops: context after the turn; total_tokens re-counts earlier passes' completions.+ context_tokens?: number;// cache_creation is Anthropic's cache-write count, cache_write_tokens OpenRouter's.prompt_tokens_details?: {cached_tokens?: number;@@ -8282,6 +8284,9 @@ export function createOpenAIStreamAdapter(promptTokens: meta.usage.prompt_tokens,completionTokens: meta.usage.completion_tokens,totalTokens: meta.usage.total_tokens,+ ...(typeof meta.usage.context_tokens === "number"+ ? { contextTokens: meta.usage.context_tokens }+ : {}),cachedTokens,cacheWriteTokens,};@@ -8466,6 +8471,9 @@ export function createOpenAIStreamAdapter(promptTokens: meta.usage.prompt_tokens,completionTokens: meta.usage.completion_tokens,totalTokens: meta.usage.total_tokens,+ ...(typeof meta.usage.context_tokens === "number"+ ? { contextTokens: meta.usage.context_tokens }+ : {}),cachedTokens,cacheWriteTokens,modelId: params.checkpoint,diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsxindex b56ea9a15b6..f7d708023dd 100644--- a/studio/frontend/src/features/chat/chat-page.tsx+++ b/studio/frontend/src/features/chat/chat-page.tsx@@ -455,7 +455,7 @@ function savedUsageFor(}// llama.cpp stops at the window, so a count past it is stale; MLX runs past it, so its count stands.const limit = store.loadedIsGguf ? store.loadedContextLength : null;- if (typeof limit === "number" && limit > 0 && (usage.totalTokens ?? 0) > limit) {+ if (typeof limit === "number" && limit > 0 && (usage.contextTokens ?? usage.totalTokens ?? 0) > limit) {return null;}return usage;@@ -4714,13 +4714,18 @@ export function ChatPage({view.mode === "single" &&(contextUsage || contextWindowKnown) ? (<ContextUsageBar- used={contextUsage?.totalTokens ?? null}+ used={contextUsage?.contextTokens ?? contextUsage?.totalTokens ?? null}// null on external providers; the bar handles that.total={loadedContextLength}cached={contextUsage?.cachedTokens}cacheWrites={contextUsage?.cacheWriteTokens}promptTokens={contextUsage?.promptTokens}- completionTokens={contextUsage?.completionTokens}+ // A tool turn's completionTokens sums every pass; the context holds only the last one.+ completionTokens={+ contextUsage?.contextTokens !== undefined+ ? contextUsage.contextTokens - contextUsage.promptTokens+ : contextUsage?.completionTokens+ }isMlx={isServedByMlx(Boolean(loadedIsGguf),platformDeviceType,diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsxindex fb5741d0e87..978cbfbfb2a 100644--- a/studio/frontend/src/features/chat/runtime-provider.tsx+++ b/studio/frontend/src/features/chat/runtime-provider.tsx@@ -2700,6 +2700,7 @@ function useStudioRuntimeAdapters(promptTokens: number;completionTokens: number;totalTokens: number;+ contextTokens?: number;cachedTokens: number;cacheWriteTokens?: number;modelId?: string;@@ -2711,7 +2712,7 @@ function useStudioRuntimeAdapters(// MLX runs past it by design, and a thread whose recount is unsupported would never get another.const localLimit = store.loadedIsGguf ? store.loadedContextLength : null;const withinLocalLimit =- !localLimit || (savedUsage?.totalTokens ?? 0) <= localLimit;+ !localLimit || (savedUsage?.contextTokens ?? savedUsage?.totalTokens ?? 0) <= localLimit;// Legacy unscoped usage (no modelId) is trusted only when a known local// window bounds the totals, so an old local turn can't be misattributed// to a newly-selected external provider.diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.tsindex dea9b30d3ea..e856d435e6a 100644--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts@@ -2232,6 +2232,8 @@ type ContextUsageSnapshot = {promptTokens: number;completionTokens: number;totalTokens: number;+ // Studio tool loops only: what the context holds (totalTokens re-counts earlier passes' output).+ contextTokens?: number;cachedTokens: number;// Anthropic-only; optional so pre-cache-stats persisted entries load.cacheWriteTokens?: number;diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.tsindex d556e10b061..6d001e17b9a 100644--- a/studio/frontend/src/features/chat/types/api.ts+++ b/studio/frontend/src/features/chat/types/api.ts@@ -788,6 +788,8 @@ export interface OpenAIChatChunk {prompt_tokens: number;completion_tokens: number;total_tokens: number;+ /** Studio tool-loop addition: the context after the turn, for usage bars. */+ context_tokens?: number;};timings?: Record<string, number>;/** Studio heuristic: the response may have stopped mid-quote. */diff --git a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts b/studio/frontend/src/features/chat/utils/chat-generation-recovery.tsindex c5eb0002c14..cd4756fd153 100644--- a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts+++ b/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts@@ -151,6 +151,7 @@ type RecoveryUsage = {prompt_tokens?: unknown;completion_tokens?: unknown;total_tokens?: unknown;+ context_tokens?: unknown;prompt_tokens_details?: { cached_tokens?: unknown; cache_write_tokens?: unknown };cache_creation_input_tokens?: unknown;cache_read_input_tokens?: unknown;@@ -218,6 +219,9 @@ export function recoveredGenerationFinalMetadata(options: {promptTokens: usage.prompt_tokens,completionTokens,totalTokens: usage.total_tokens,+ ...(typeof usage.context_tokens === "number"+ ? { contextTokens: usage.context_tokens }+ : {}),cachedTokens:(typeof timings?.cache_n === "number" ? timings.cache_n : undefined) ??(typeof usage.prompt_tokens_details?.cached_tokens === "number"diff --git a/studio/frontend/tests/chat-runtime-reload-status.test.ts b/studio/frontend/tests/chat-runtime-reload-status.test.tsindex 0e66cfb467e..5fa69760706 100644--- a/studio/frontend/tests/chat-runtime-reload-status.test.ts+++ b/studio/frontend/tests/chat-runtime-reload-status.test.ts@@ -168,6 +168,34 @@ test("terminal recovery reads cache writes from either provider's usage shape",);});+test("terminal recovery prices a tool loop's context, not its summed completions", () => {+ const metadata = recoveredGenerationFinalMetadata({+ current: { generationSettled: true },+ run: {+ id: "run-tools",+ requestPayload: { model: "local/model" },+ createdAt: 100,+ startedAt: 120,+ completedAt: 1120,+ },+ usage: {+ prompt_tokens: 140,+ completion_tokens: 40,+ total_tokens: 180,+ context_tokens: 150,+ },+ timings: {},+ firstChunkAt: 220,+ totalChunks: 4,+ });+ const usage = metadata.contextUsage as {+ totalTokens: number;+ contextTokens: number;+ };+ assert.equal(usage.contextTokens, 150);+ assert.equal(usage.totalTokens, 180);+});+test("terminal recovery restores final local usage and timing metadata", () => {const metadata = recoveredGenerationFinalMetadata({current: { generationSettled: true },diff --git a/studio/frontend/tests/context-window-visible.test.ts b/studio/frontend/tests/context-window-visible.test.tsindex bf46a3d2f8f..7f15477daa0 100644--- a/studio/frontend/tests/context-window-visible.test.ts+++ b/studio/frontend/tests/context-window-visible.test.ts@@ -151,6 +151,9 @@ test("the header renders the bar on the window alone, with usage optional", () =page,/showContextWindowUsage &&\s*view\.mode === "single" &&\s*\(contextUsage \|\| contextWindowKnown\)/,);- assert.match(page, /used=\{contextUsage\?\.totalTokens \?\? null\}/);+ assert.match(+ page,+ /used=\{contextUsage\?\.contextTokens \?\? contextUsage\?\.totalTokens \?\? null\}/,+ );});