Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: fix the context meter after tool calls on MLX and transformers (#13913)
Nilay8d85be61cee0
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex 3aa0641291d..504387d80a5 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -347,15 +347,21 @@ def _summed_tool_loop_stats(total, turn):return turnprior_usage = total.get("usage") or {}usage = dict(turn.get("usage") or {})+ # Earlier turns' completions are inside this prompt, so the context adds this turn's only.+ if usage.get("prompt_tokens"):+ usage["context_tokens"] = usage["prompt_tokens"] + (usage.get("completion_tokens") or 0)completion = (usage.get("completion_tokens") or 0) + (prior_usage.get("completion_tokens") or 0)usage["completion_tokens"] = completion# The prompt is the loop's, not one turn's, so a turn that ended before reporting keeps the last count that- # arrived. Its details describe that same count and move with it, or cached tokens could outnumber prompt tokens.+ # arrived. Its details describe that same count and move with it, or cached tokens could outnumber prompt tokens;+ # so does the context priced from it.if not usage.get("prompt_tokens"):usage["prompt_tokens"] = prior_usage.get("prompt_tokens") or 0usage.pop("prompt_tokens_details", None)if prior_usage.get("prompt_tokens_details") is not None:usage["prompt_tokens_details"] = prior_usage["prompt_tokens_details"]+ if prior_usage.get("context_tokens") is not None:+ usage["context_tokens"] = prior_usage["context_tokens"]usage["total_tokens"] = usage["prompt_tokens"] + completion# Details describe the completion, so they sum with it rather than describing one turn against every turn's tokensdetails = dict(prior_usage.get("completion_tokens_details") or {})diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 617c5dfdb3c..4cd9aa5816f 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -46704,6 +46704,8 @@ def _sum_passthrough_attempt_stats(first: dict, retry: dict, selected: dict) ->aggregate = _summed_tool_loop_stats(first, retry)result = dict(selected)result["usage"] = aggregate["usage"]+ # Two attempts at one reply, not tool passes: the client gets llama-server's usage shape back.+ result["usage"].pop("context_tokens", None)if "timings" in aggregate:result["timings"] = aggregate["timings"]return resultdiff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex 572a1bdea85..86ae26a7de8 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -1358,7 +1358,12 @@ def test_every_tool_loop_turn_is_billed_not_just_the_last():earlier results."""folded = _fold(_turn(100, 20), _turn(160, 30), _turn(220, 5))- assert folded["usage"] == {"prompt_tokens": 220, "completion_tokens": 55, "total_tokens": 275}+ assert folded["usage"] == {+ "prompt_tokens": 220,+ "completion_tokens": 55,+ "total_tokens": 275,+ "context_tokens": 225,+ }# Rates describe the summed counts, not the last turn that arrived.assert folded["timings"]["predicted_n"] == 55assert folded["timings"]["predicted_ms"] == pytest.approx(550.0)@@ -1380,6 +1385,17 @@ def test_a_turn_that_ends_before_reporting_does_not_erase_the_loop():assert errored["usage"] == {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}+def test_the_context_meter_counts_earlier_turns_once():+ folded = _fold(_turn(18000, 60), _turn(21500, 80), _turn(25000, 700))++ assert folded["usage"]["total_tokens"] == 25840+ assert folded["usage"]["context_tokens"] == 25700+ assert "context_tokens" not in _fold(_turn(18000, 60))["usage"]+ # A last report with no prompt count keeps the earlier context.+ unreported = _fold(_turn(18000, 60), _turn(21500, 80), {"finish_reason": "stop"})+ assert unreported["usage"]["context_tokens"] == 21580++def test_completion_details_are_summed_with_the_completion_they_describe():"""Carrying the last turn's details would report them against every turn'stokens."""