Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: fix the context meter after tool calls on MLX and transformers (#13913)

Nilay8d85be61cee0
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index 3aa0641291d..504387d80a5 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -347,15 +347,21 @@ def _summed_tool_loop_stats(total, turn):
return turn
prior_usage = total.get("usage") or {}
usage = dict(turn.get("usage") or {})
+ # Earlier turns' completions are inside this prompt, so the context adds this turn's only.
+ if usage.get("prompt_tokens"):
+ usage["context_tokens"] = usage["prompt_tokens"] + (usage.get("completion_tokens") or 0)
completion = (usage.get("completion_tokens") or 0) + (prior_usage.get("completion_tokens") or 0)
usage["completion_tokens"] = completion
# The prompt is the loop's, not one turn's, so a turn that ended before reporting keeps the last count that
- # arrived. Its details describe that same count and move with it, or cached tokens could outnumber prompt tokens.
+ # arrived. Its details describe that same count and move with it, or cached tokens could outnumber prompt tokens;
+ # so does the context priced from it.
if not usage.get("prompt_tokens"):
usage["prompt_tokens"] = prior_usage.get("prompt_tokens") or 0
usage.pop("prompt_tokens_details", None)
if prior_usage.get("prompt_tokens_details") is not None:
usage["prompt_tokens_details"] = prior_usage["prompt_tokens_details"]
+ if prior_usage.get("context_tokens") is not None:
+ usage["context_tokens"] = prior_usage["context_tokens"]
usage["total_tokens"] = usage["prompt_tokens"] + completion
# Details describe the completion, so they sum with it rather than describing one turn against every turn's tokens
details = dict(prior_usage.get("completion_tokens_details") or {})
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 617c5dfdb3c..4cd9aa5816f 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -46704,6 +46704,8 @@ def _sum_passthrough_attempt_stats(first: dict, retry: dict, selected: dict) ->
aggregate = _summed_tool_loop_stats(first, retry)
result = dict(selected)
result["usage"] = aggregate["usage"]
+ # Two attempts at one reply, not tool passes: the client gets llama-server's usage shape back.
+ result["usage"].pop("context_tokens", None)
if "timings" in aggregate:
result["timings"] = aggregate["timings"]
return result
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index 572a1bdea85..86ae26a7de8 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -1358,7 +1358,12 @@ def test_every_tool_loop_turn_is_billed_not_just_the_last():
earlier results."""
folded = _fold(_turn(100, 20), _turn(160, 30), _turn(220, 5))
- assert folded["usage"] == {"prompt_tokens": 220, "completion_tokens": 55, "total_tokens": 275}
+ assert folded["usage"] == {
+ "prompt_tokens": 220,
+ "completion_tokens": 55,
+ "total_tokens": 275,
+ "context_tokens": 225,
+ }
# Rates describe the summed counts, not the last turn that arrived.
assert folded["timings"]["predicted_n"] == 55
assert folded["timings"]["predicted_ms"] == pytest.approx(550.0)
@@ -1380,6 +1385,17 @@ def test_a_turn_that_ends_before_reporting_does_not_erase_the_loop():
assert errored["usage"] == {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120}
+def test_the_context_meter_counts_earlier_turns_once():
+ folded = _fold(_turn(18000, 60), _turn(21500, 80), _turn(25000, 700))
+
+ assert folded["usage"]["total_tokens"] == 25840
+ assert folded["usage"]["context_tokens"] == 25700
+ assert "context_tokens" not in _fold(_turn(18000, 60))["usage"]
+ # A last report with no prompt count keeps the earlier context.
+ unreported = _fold(_turn(18000, 60), _turn(21500, 80), {"finish_reason": "stop"})
+ assert unreported["usage"]["context_tokens"] == 21580
+
+
def test_completion_details_are_summed_with_the_completion_they_describe():
"""Carrying the last turn's details would report them against every turn's
tokens."""