Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: price the context meter off a tool loop's final pass, not the whole turn's completions (#12889)

SlippyChimpMG06bbb7f90bf7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 810cf74458f..41777084028 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -36424,6 +36424,9 @@ class LlamaCppBackend:
"completion_tokens": _tc,
"total_tokens": _fp + _tc,
}
+ # Earlier passes' completions are already inside _fp, so the context is _fp + this pass only.
+ if _accumulated_completion_tokens:
+ _usage["context_tokens"] = _fp + int(_fu.get("completion_tokens") or 0)
# Preserve KV-cache hit details (cached_tokens) so the tool path
# reports them like the standard non-tool path does, not always 0.
if _fu.get("prompt_tokens_details"):
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index 98395f07d45..44b441695e3 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -19,6 +19,7 @@ from pydantic import (
PrivateAttr,
Tag,
field_validator,
+ model_serializer,
model_validator,
)
@@ -3456,6 +3457,8 @@ class CompletionUsage(BaseModel):
prompt_tokens: int = 0
completion_tokens: int = 0
total_tokens: int = 0
+ # Studio tool loops: final pass's prompt + completion (total_tokens sums every pass's completion).
+ context_tokens: Optional[int] = None
prompt_tokens_details: Optional[dict] = Field(
default_factory = lambda: {"cached_tokens": 0, "audio_tokens": 0}
)
@@ -3468,6 +3471,14 @@ class CompletionUsage(BaseModel):
}
)
+ @model_serializer(mode = "wrap")
+ def _omit_unset_context_tokens(self, handler):
+ # Only tool loops set it; elsewhere the OpenAI usage object stays byte-identical.
+ data = handler(self)
+ if isinstance(data, dict) and data.get("context_tokens") is None:
+ data.pop("context_tokens", None)
+ return data
+
class ChatCompletion(BaseModel):
"""Non-streaming chat completion response."""
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 51b8b312d63..9523381a068 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1474,6 +1474,7 @@ def _openai_stream_usage_chunk(
completion_tokens = _completion_tokens,
total_tokens = _total_tokens,
prompt_tokens_details = _prompt_tokens_details(_usage.get("prompt_tokens_details")),
+ context_tokens = _usage.get("context_tokens"),
),
timings = stream_timings,
)
diff --git a/studio/backend/tests/test_llama_cpp_tool_loop.py b/studio/backend/tests/test_llama_cpp_tool_loop.py
index 0cf2745a680..567a76ede62 100644
--- a/studio/backend/tests/test_llama_cpp_tool_loop.py
+++ b/studio/backend/tests/test_llama_cpp_tool_loop.py
@@ -4521,6 +4521,21 @@ def test_metadata_event_preserves_prompt_tokens_details(monkeypatch):
assert usage["completion_tokens"] == 4
+def test_metadata_event_omits_context_tokens_on_a_single_pass(monkeypatch):
+ stream = [
+ _sse({"content": "hi"}),
+ _usage_done({"prompt_tokens": 5, "completion_tokens": 2}),
+ _done(),
+ ]
+ backend, _ = _backend_and_payloads(monkeypatch, [stream])
+
+ events = _run_tool_loop(backend, [{"role": "user", "content": "hi"}], [_web_search_tool()])
+
+ usage = [e for e in events if e.get("type") == "metadata"][-1]["usage"]
+ assert usage["total_tokens"] == 7
+ assert "context_tokens" not in usage
+
+
def test_metadata_event_omits_prompt_tokens_details_when_absent(monkeypatch):
"""No KV-cache block from the server -> the key isn't fabricated, so the
route falls back to its 0-default instead of reading a bogus value."""
@@ -4538,6 +4553,39 @@ def test_metadata_event_omits_prompt_tokens_details_when_absent(monkeypatch):
assert "prompt_tokens_details" not in metadata[-1]["usage"]
+def test_metadata_event_context_tokens_count_earlier_passes_once(monkeypatch):
+ """A tool pass's completion is re-sent inside the next pass's prompt, so the
+ context the turn leaves is the final prompt plus the final completion only;
+ total_tokens keeps billing every pass's completion."""
+ streams = [
+ [
+ _tool_call_sse("web_search", {"query": "cats"}, "call_1"),
+ _usage_done({"prompt_tokens": 100, "completion_tokens": 30}, "tool_calls"),
+ _done(),
+ ],
+ [
+ _sse({"content": "Found cats."}),
+ _usage_done({"prompt_tokens": 140, "completion_tokens": 10}),
+ _done(),
+ ],
+ ]
+ backend, payloads = _backend_and_payloads(monkeypatch, streams)
+ _record_tool_calls(monkeypatch, "result")
+
+ events = _run_tool_loop(
+ backend,
+ [{"role": "user", "content": "search cats"}],
+ [_web_search_tool()],
+ max_tool_iterations = 2,
+ )
+
+ assert len(payloads) == 2
+ usage = [e for e in events if e.get("type") == "metadata"][-1]["usage"]
+ assert usage["completion_tokens"] == 40
+ assert usage["total_tokens"] == 180
+ assert usage["context_tokens"] == 150
+
+
def test_gguf_rehearsal_name_split_before_args_is_not_leaked(monkeypatch):
"""Finding 6: a rehearsal call whose name (``web_search``) and ``[ARGS]{...}``
arrive in separate content deltas must hold the bare name in the buffer until
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index 075b4b3128e..0ffcb617831 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -3437,6 +3437,32 @@ class TestOpenAICompatibilityHelpers:
assert usage["completion_tokens"] == 7
assert usage["total_tokens"] == 7
+ def test_stream_usage_chunk_forwards_tool_loop_context_tokens(self):
+ payload = SimpleNamespace(stream_options = {"include_usage": True})
+ tool_loop = {
+ "prompt_tokens": 140,
+ "completion_tokens": 40,
+ "total_tokens": 180,
+ "context_tokens": 150,
+ }
+ line = _openai_stream_usage_chunk(payload, "c", 1, "m", tool_loop, None)
+ usage = json.loads(line.removeprefix("data: "))["usage"]
+ assert usage["total_tokens"] == 180
+ assert usage["context_tokens"] == 150
+
+ single = {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}
+ line = _openai_stream_usage_chunk(payload, "c", 1, "m", single, None)
+ assert "context_tokens" not in json.loads(line.removeprefix("data: "))["usage"]
+
+ def test_non_streaming_usage_omits_unset_context_tokens(self):
+ from models.inference import CompletionUsage
+
+ assert "context_tokens" not in json.loads(CompletionUsage(total_tokens = 5).model_dump_json())
+ assert "context_tokens" not in CompletionUsage(total_tokens = 5).model_dump()
+ assert (
+ json.loads(CompletionUsage(context_tokens = 4).model_dump_json())["context_tokens"] == 4
+ )
+
def test_completion_stream_monitor_reads_usage_before_client_strip(self, monkeypatch):
import routes.inference as inf_mod
diff --git a/studio/frontend/src/features/chat/api/chat-adapter.ts b/studio/frontend/src/features/chat/api/chat-adapter.ts
index 7c223695267..4128888811d 100644
--- a/studio/frontend/src/features/chat/api/chat-adapter.ts
+++ b/studio/frontend/src/features/chat/api/chat-adapter.ts
@@ -455,6 +455,8 @@ interface ServerUsage {
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
+ // Studio tool loops: context after the turn; total_tokens re-counts earlier passes' completions.
+ context_tokens?: number;
// cache_creation is Anthropic's cache-write count, cache_write_tokens OpenRouter's.
prompt_tokens_details?: {
cached_tokens?: number;
@@ -8282,6 +8284,9 @@ export function createOpenAIStreamAdapter(
promptTokens: meta.usage.prompt_tokens,
completionTokens: meta.usage.completion_tokens,
totalTokens: meta.usage.total_tokens,
+ ...(typeof meta.usage.context_tokens === "number"
+ ? { contextTokens: meta.usage.context_tokens }
+ : {}),
cachedTokens,
cacheWriteTokens,
};
@@ -8466,6 +8471,9 @@ export function createOpenAIStreamAdapter(
promptTokens: meta.usage.prompt_tokens,
completionTokens: meta.usage.completion_tokens,
totalTokens: meta.usage.total_tokens,
+ ...(typeof meta.usage.context_tokens === "number"
+ ? { contextTokens: meta.usage.context_tokens }
+ : {}),
cachedTokens,
cacheWriteTokens,
modelId: params.checkpoint,
diff --git a/studio/frontend/src/features/chat/chat-page.tsx b/studio/frontend/src/features/chat/chat-page.tsx
index b56ea9a15b6..f7d708023dd 100644
--- a/studio/frontend/src/features/chat/chat-page.tsx
+++ b/studio/frontend/src/features/chat/chat-page.tsx
@@ -455,7 +455,7 @@ function savedUsageFor(
}
// llama.cpp stops at the window, so a count past it is stale; MLX runs past it, so its count stands.
const limit = store.loadedIsGguf ? store.loadedContextLength : null;
- if (typeof limit === "number" && limit > 0 && (usage.totalTokens ?? 0) > limit) {
+ if (typeof limit === "number" && limit > 0 && (usage.contextTokens ?? usage.totalTokens ?? 0) > limit) {
return null;
}
return usage;
@@ -4714,13 +4714,18 @@ export function ChatPage({
view.mode === "single" &&
(contextUsage || contextWindowKnown) ? (
<ContextUsageBar
- used={contextUsage?.totalTokens ?? null}
+ used={contextUsage?.contextTokens ?? contextUsage?.totalTokens ?? null}
// null on external providers; the bar handles that.
total={loadedContextLength}
cached={contextUsage?.cachedTokens}
cacheWrites={contextUsage?.cacheWriteTokens}
promptTokens={contextUsage?.promptTokens}
- completionTokens={contextUsage?.completionTokens}
+ // A tool turn's completionTokens sums every pass; the context holds only the last one.
+ completionTokens={
+ contextUsage?.contextTokens !== undefined
+ ? contextUsage.contextTokens - contextUsage.promptTokens
+ : contextUsage?.completionTokens
+ }
isMlx={isServedByMlx(
Boolean(loadedIsGguf),
platformDeviceType,
diff --git a/studio/frontend/src/features/chat/runtime-provider.tsx b/studio/frontend/src/features/chat/runtime-provider.tsx
index fb5741d0e87..978cbfbfb2a 100644
--- a/studio/frontend/src/features/chat/runtime-provider.tsx
+++ b/studio/frontend/src/features/chat/runtime-provider.tsx
@@ -2700,6 +2700,7 @@ function useStudioRuntimeAdapters(
promptTokens: number;
completionTokens: number;
totalTokens: number;
+ contextTokens?: number;
cachedTokens: number;
cacheWriteTokens?: number;
modelId?: string;
@@ -2711,7 +2712,7 @@ function useStudioRuntimeAdapters(
// MLX runs past it by design, and a thread whose recount is unsupported would never get another.
const localLimit = store.loadedIsGguf ? store.loadedContextLength : null;
const withinLocalLimit =
- !localLimit || (savedUsage?.totalTokens ?? 0) <= localLimit;
+ !localLimit || (savedUsage?.contextTokens ?? savedUsage?.totalTokens ?? 0) <= localLimit;
// Legacy unscoped usage (no modelId) is trusted only when a known local
// window bounds the totals, so an old local turn can't be misattributed
// to a newly-selected external provider.
diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
index dea9b30d3ea..e856d435e6a 100644
--- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
+++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts
@@ -2232,6 +2232,8 @@ type ContextUsageSnapshot = {
promptTokens: number;
completionTokens: number;
totalTokens: number;
+ // Studio tool loops only: what the context holds (totalTokens re-counts earlier passes' output).
+ contextTokens?: number;
cachedTokens: number;
// Anthropic-only; optional so pre-cache-stats persisted entries load.
cacheWriteTokens?: number;
diff --git a/studio/frontend/src/features/chat/types/api.ts b/studio/frontend/src/features/chat/types/api.ts
index d556e10b061..6d001e17b9a 100644
--- a/studio/frontend/src/features/chat/types/api.ts
+++ b/studio/frontend/src/features/chat/types/api.ts
@@ -788,6 +788,8 @@ export interface OpenAIChatChunk {
prompt_tokens: number;
completion_tokens: number;
total_tokens: number;
+ /** Studio tool-loop addition: the context after the turn, for usage bars. */
+ context_tokens?: number;
};
timings?: Record<string, number>;
/** Studio heuristic: the response may have stopped mid-quote. */
diff --git a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts b/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts
index c5eb0002c14..cd4756fd153 100644
--- a/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts
+++ b/studio/frontend/src/features/chat/utils/chat-generation-recovery.ts
@@ -151,6 +151,7 @@ type RecoveryUsage = {
prompt_tokens?: unknown;
completion_tokens?: unknown;
total_tokens?: unknown;
+ context_tokens?: unknown;
prompt_tokens_details?: { cached_tokens?: unknown; cache_write_tokens?: unknown };
cache_creation_input_tokens?: unknown;
cache_read_input_tokens?: unknown;
@@ -218,6 +219,9 @@ export function recoveredGenerationFinalMetadata(options: {
promptTokens: usage.prompt_tokens,
completionTokens,
totalTokens: usage.total_tokens,
+ ...(typeof usage.context_tokens === "number"
+ ? { contextTokens: usage.context_tokens }
+ : {}),
cachedTokens:
(typeof timings?.cache_n === "number" ? timings.cache_n : undefined) ??
(typeof usage.prompt_tokens_details?.cached_tokens === "number"
diff --git a/studio/frontend/tests/chat-runtime-reload-status.test.ts b/studio/frontend/tests/chat-runtime-reload-status.test.ts
index 0e66cfb467e..5fa69760706 100644
--- a/studio/frontend/tests/chat-runtime-reload-status.test.ts
+++ b/studio/frontend/tests/chat-runtime-reload-status.test.ts
@@ -168,6 +168,34 @@ test("terminal recovery reads cache writes from either provider's usage shape",
);
});
+test("terminal recovery prices a tool loop's context, not its summed completions", () => {
+ const metadata = recoveredGenerationFinalMetadata({
+ current: { generationSettled: true },
+ run: {
+ id: "run-tools",
+ requestPayload: { model: "local/model" },
+ createdAt: 100,
+ startedAt: 120,
+ completedAt: 1120,
+ },
+ usage: {
+ prompt_tokens: 140,
+ completion_tokens: 40,
+ total_tokens: 180,
+ context_tokens: 150,
+ },
+ timings: {},
+ firstChunkAt: 220,
+ totalChunks: 4,
+ });
+ const usage = metadata.contextUsage as {
+ totalTokens: number;
+ contextTokens: number;
+ };
+ assert.equal(usage.contextTokens, 150);
+ assert.equal(usage.totalTokens, 180);
+});
+
test("terminal recovery restores final local usage and timing metadata", () => {
const metadata = recoveredGenerationFinalMetadata({
current: { generationSettled: true },
diff --git a/studio/frontend/tests/context-window-visible.test.ts b/studio/frontend/tests/context-window-visible.test.ts
index bf46a3d2f8f..7f15477daa0 100644
--- a/studio/frontend/tests/context-window-visible.test.ts
+++ b/studio/frontend/tests/context-window-visible.test.ts
@@ -151,6 +151,9 @@ test("the header renders the bar on the window alone, with usage optional", () =
page,
/showContextWindowUsage &&\s*view\.mode === "single" &&\s*\(contextUsage \|\| contextWindowKnown\)/,
);
- assert.match(page, /used=\{contextUsage\?\.totalTokens \?\? null\}/);
+ assert.match(
+ page,
+ /used=\{contextUsage\?\.contextTokens \?\? contextUsage\?\.totalTokens \?\? null\}/,
+ );
});