Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

studio: use generation usage for conversation search budgets (#9438)

Maheswar Kumar6e21a68ebdfe
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 72bf5884fde..c3ffe241dad 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -973,30 +973,39 @@ class InferenceBackend:
from core.inference.safetensors_agentic import run_safetensors_tool_loop
from core.inference.tools import execute_tool
+ # The usage of the LATEST turn, so the loop can size a search against a real
+ # token count. Cleared before every turn and written when that turn ends, so a
+ # turn that failed or reported nothing leaves it empty rather than stale.
+ _turn_stats: dict = {}
+
def _single_turn(conv: list, *, active_tools: Optional[list[dict]] = None):
# conv already has the system message -- avoid double-prepend.
# `active_tools` is supplied by run_safetensors_tool_loop so one-shot
# tools such as render_html can be removed from later same-response prompts.
turn_tools = active_tools if active_tools is not None else tools
- yield from self._generate_chat_response_inner(
- messages = conv,
- system_prompt = "",
- temperature = temperature,
- top_p = top_p,
- top_k = top_k,
- min_p = min_p,
- max_new_tokens = max_new_tokens,
- repetition_penalty = repetition_penalty,
- cancel_event = cancel_event,
- tools = turn_tools,
- enable_thinking = enable_thinking,
- reasoning_effort = reasoning_effort,
- preserve_thinking = preserve_thinking,
- # Self-limiting: after a tool call the conversation ends on a tool
- # result, so later turns render as ordinary new turns.
- continue_final_message = continue_final_message,
- presence_penalty = presence_penalty,
- )
+ _turn_stats["stats"] = None
+ try:
+ yield from self._generate_chat_response_inner(
+ messages = conv,
+ system_prompt = "",
+ temperature = temperature,
+ top_p = top_p,
+ top_k = top_k,
+ min_p = min_p,
+ max_new_tokens = max_new_tokens,
+ repetition_penalty = repetition_penalty,
+ cancel_event = cancel_event,
+ tools = turn_tools,
+ enable_thinking = enable_thinking,
+ reasoning_effort = reasoning_effort,
+ preserve_thinking = preserve_thinking,
+ # Self-limiting: after a tool call the conversation ends on a tool
+ # result, so later turns render as ordinary new turns.
+ continue_final_message = continue_final_message,
+ presence_penalty = presence_penalty,
+ )
+ finally:
+ _turn_stats["stats"] = self.last_generation_stats
initial = list(messages)
if system_prompt:
@@ -1042,6 +1051,7 @@ class InferenceBackend:
# So a conversation search can be sized against what this model can hold.
context_length = _model_info.get("context_length"),
max_tokens = max_new_tokens,
+ generation_stats_holder = _turn_stats,
)
def generate_chat_response(
diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.py
index d1e3c27a2a5..a6b5f93833d 100644
--- a/studio/backend/core/inference/orchestrator.py
+++ b/studio/backend/core/inference/orchestrator.py
@@ -2206,12 +2206,18 @@ class InferenceOrchestrator:
max_new_tokens = max_tokens if max_tokens and max_tokens > 0 else 2048
+ # The worker's usage for the LATEST turn only. Hoisted out of the turn so the
+ # loop can size a conversation search against a real prompt count, and cleared
+ # on the way in rather than on each way out, so a turn that failed or was
+ # cancelled leaves it empty instead of handing the loop an earlier turn's number.
+ turn_stats: dict = {}
+
def _single_turn(conv: list, *, active_tools: Optional[list[dict]] = None):
# ``conv`` already carries any system message. ``active_tools`` lets
# run_safetensors_tool_loop drop one-shot tools (e.g. render_html) from
# later same-response prompts.
turn_tools = active_tools if active_tools is not None else tools
- turn_stats: dict = {}
+ turn_stats.clear()
common_kwargs = dict(
messages = conv,
system_prompt = "",
@@ -2315,6 +2321,7 @@ class InferenceOrchestrator:
# So a conversation search can be sized against what this model can hold.
context_length = _model_info.get("context_length"),
max_tokens = max_new_tokens,
+ generation_stats_holder = turn_stats,
)
def generate_with_adapter_control(
diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.py
index 8eb31e1afa4..97be8397b5b 100644
--- a/studio/backend/core/inference/safetensors_agentic.py
+++ b/studio/backend/core/inference/safetensors_agentic.py
@@ -470,6 +470,47 @@ def _call_single_turn(single_turn, conversation: list, active_tools: list[dict])
return single_turn(conversation)
+def _dense_message_tokens(messages: list[dict]) -> int:
+ """`estimate_messages_tokens_dense`, imported where it is used like the rest here."""
+ from core.inference.context_window import estimate_messages_tokens_dense
+ return estimate_messages_tokens_dense(messages)
+
+
+def _spent_prompt_tokens(
+ conversation: list[dict],
+ tools: Optional[list[dict]],
+ generation_stats_holder: Optional[dict],
+ prompt_dense_tokens: int,
+) -> int:
+ """Tokens the next prompt already owes, from the count the last turn reported.
+
+ The backend tokenises the turn's prompt to run it and ships that count on gen_done,
+ tool catalogue included, so the only part left to estimate is what the loop appended
+ afterwards: this turn's assistant text and the results of any tool already run in the
+ same batch. Estimating that tail alone is what stops a long English preamble being
+ charged several times what it costs.
+
+ An exact recount here is not available. This loop runs in the PARENT process and
+ `InferenceOrchestrator.models` mirrors the worker's model_info, which carries no
+ tokenizer, so counting the conversation again would mean a round trip into the worker
+ between every tool call.
+
+ Without a report the estimate covers the whole thread, as it did before. Dense
+ because four characters per token undercounts CJK and emoji by about half: measured
+ on an 81-message CJK chat, 1295 estimated against 2737 real, reporting 1777 tokens of
+ room where 335 remained. Never floored to zero, which reaches the tool as "there is
+ no room left in this context to search earlier conversation" and switches recall off
+ on exactly the tight windows that need it.
+ """
+ stats = (generation_stats_holder or {}).get("stats")
+ usage = stats.get("usage") if isinstance(stats, dict) else None
+ prompt_tokens = usage.get("prompt_tokens") if isinstance(usage, dict) else None
+ if isinstance(prompt_tokens, int) and not isinstance(prompt_tokens, bool) and prompt_tokens > 0:
+ added = _dense_message_tokens(conversation) - prompt_dense_tokens
+ return prompt_tokens + max(0, added)
+ return _dense_message_tokens(conversation) + _dense_message_tokens(tools or [])
+
+
def run_safetensors_tool_loop(
*,
single_turn: Callable[[list], Generator[str, None, None]],
@@ -493,6 +534,7 @@ def run_safetensors_tool_loop(
renderable_tools = None,
context_length: Optional[int] = None,
max_tokens: Optional[int] = None,
+ generation_stats_holder: Optional[dict] = None,
) -> Generator[dict, None, None]:
"""Drive an agentic tool loop on top of a cumulative-text generator.
@@ -698,6 +740,9 @@ def run_safetensors_tool_loop(
return False
return _first_detected_tool_name(content) == "render_html"
+ # The conversation as this turn's prompt renders it, so what the loop appends
+ # afterwards can be charged on its own against the count the turn reports.
+ prompt_dense_tokens = _dense_message_tokens(conversation)
gen = _call_single_turn(single_turn, conversation, active_tools)
prev_cumulative = ""
@@ -1325,26 +1370,27 @@ def run_safetensors_tool_loop(
if context_length and _accepts_kwarg(
execute_tool, "conversation_budget_tokens"
):
- from core.inference.context_window import (
- estimate_messages_tokens_dense,
- retrieval_budget,
- )
+ from core.inference.context_window import retrieval_budget
- # Dense, unlike the eviction estimator: four characters per token
- # undercounts CJK and emoji by about half, and this path has no
- # rolling fit to recover if the tool exchange it sizes then puts
- # the next prompt over the window. Measured on an 81-message CJK
- # chat: 1295 estimated against 2737 real, reporting 1777 tokens of
- # room where 335 remained.
+ # From the tokenizer count the last turn reported rather than
+ # from characters alone: `conversation` already carries this
+ # turn's assistant preamble and its tool call, and pricing the
+ # whole thread by characters is what hands a retrieval room the
+ # next prompt does not have.
#
# `reply_returns`, as the GGUF loop does: result and reply are
# both protected on the next fit, so one retrieval cannot spend
# the budget they share.
+ spent = _spent_prompt_tokens(
+ conversation,
+ tools,
+ generation_stats_holder,
+ prompt_dense_tokens,
+ )
kwargs["conversation_budget_tokens"] = retrieval_budget(
int(context_length),
max_tokens,
- estimate_messages_tokens_dense(conversation)
- + estimate_messages_tokens_dense(tools or []),
+ spent,
reply_returns = True,
)
# And what a RESULT may add, which is the same question asked of
diff --git a/studio/backend/tests/test_conversation_search_safetensors_loop.py b/studio/backend/tests/test_conversation_search_safetensors_loop.py
index f9316877f9f..3f8d7e7a641 100644
--- a/studio/backend/tests/test_conversation_search_safetensors_loop.py
+++ b/studio/backend/tests/test_conversation_search_safetensors_loop.py
@@ -116,9 +116,7 @@ def test_the_budget_charges_token_dense_text_at_its_real_rate():
single_turn = _searching_model(1),
messages = list(dense_messages),
tools = tools,
- execute_tool = lambda name, arguments, **kwargs: (
- seen.update(kwargs) or "an earlier turn"
- ),
+ execute_tool = lambda name, arguments, **kwargs: seen.update(kwargs) or "an earlier turn",
cancel_event = threading.Event(),
max_tool_iterations = 2,
thread_id = "t-sf",
@@ -180,6 +178,154 @@ def test_the_loop_budgets_its_search_against_this_models_context():
assert budget < prompt_budget(4096, 512) - estimate_messages_tokens(MESSAGES)
+def _model_that_thinks_out_loud(preamble, stats_holder, reported):
+ """A model that writes prose before its one search, reporting usage as a turn does."""
+ state = {"n": 0}
+
+ def single_turn(messages):
+ stats_holder["stats"] = None
+ state["n"] += 1
+ if state["n"] > 1:
+ yield "done"
+ return
+ call = (
+ preamble
+ + '<tool_call>{"name":"search_conversation","arguments":{"query":"the code"}}'
+ + "</tool_call>"
+ )
+ buffer = ""
+ for char in call:
+ buffer += char
+ yield buffer
+ stats_holder["stats"] = {"usage": {"prompt_tokens": reported}}
+
+ return single_turn
+
+
+def _budget_after_a_preamble(preamble, reported, stats_holder):
+ seen = {}
+ list(
+ run_safetensors_tool_loop(
+ single_turn = _model_that_thinks_out_loud(preamble, stats_holder, reported),
+ messages = [
+ {"role": "user", "content": "recall the deployment steps. " * 420},
+ {"role": "user", "content": "what was the code from earlier"},
+ ],
+ tools = [{"type": "function", "function": {"name": "search_conversation"}}],
+ execute_tool = lambda name, arguments, **kwargs: seen.update(kwargs) or "an earlier turn",
+ cancel_event = threading.Event(),
+ max_tool_iterations = 2,
+ thread_id = "t-sf",
+ context_length = 8192,
+ max_tokens = 512,
+ generation_stats_holder = stats_holder,
+ )
+ )
+ return seen["conversation_budget_tokens"]
+
+
+def test_the_budget_charges_this_turns_own_output_in_tokens_not_bytes():
+ """A 4600-character English preamble costs about 1150 tokens, not 4600.
+
+ Charged by its byte length it takes the whole prompt budget of an 8K window, so the
+ search is refused with roughly 3.4K tokens still free, which is worse than the
+ estimate it replaces. The fixture is deliberately large: on a three-line chat the
+ two numbers differ by too little to fail an assertion.
+ """
+ from core.inference.context_window import retrieval_budget
+
+ preamble = "the answer is somewhere in the earlier turns. " * 100
+ budget = _budget_after_a_preamble(preamble, 3_067, {})
+
+ # Charged as its 4600 bytes this is nothing. Charged as tokens it is a real search.
+ assert budget > 3_000
+ # And it IS charged: the room is smaller than it was before the turn wrote anything.
+ assert budget < retrieval_budget(8192, 512, 3_067, reply_returns = True)
+
+
+def test_the_budget_spends_the_count_the_turn_reported():
+ """The reported prompt count is a tokenizer's, so it outranks the character estimate.
+
+ Six thousand tokens of a window this size leaves room for a small retrieval and no
+ more; the estimate alone would price the same thread at a third of that and hand out
+ room the next prompt does not have.
+ """
+ budget = _budget_after_a_preamble("thinking. ", 6_000, {})
+
+ assert 0 < budget < 1_800
+
+
+def test_the_budget_estimates_when_no_usage_was_reported():
+ """A cancelled or unreported turn falls back to the estimate, never to zero.
+
+ Zero reaches the tool as "there is no room left in this context to search earlier
+ conversation", the same failure `_recall_top_k` carries a note about: it switches
+ recall off on exactly the tight windows that need it.
+ """
+ from core.inference.context_window import (
+ estimate_messages_tokens_dense,
+ retrieval_budget,
+ )
+
+ seen = {}
+ tools = [{"type": "function", "function": {"name": "search_conversation"}}]
+
+ list(
+ run_safetensors_tool_loop(
+ single_turn = _searching_model(1),
+ messages = list(MESSAGES),
+ tools = tools,
+ execute_tool = lambda name, arguments, **kwargs: seen.update(kwargs) or "an earlier turn",
+ cancel_event = threading.Event(),
+ max_tool_iterations = 2,
+ thread_id = "t-sf",
+ context_length = 4096,
+ max_tokens = 512,
+ generation_stats_holder = {"stats": None},
+ )
+ )
+
+ budget = seen["conversation_budget_tokens"]
+ assert budget > 0
+ assert budget <= retrieval_budget(
+ 4096, 512, estimate_messages_tokens_dense(tools), reply_returns = True
+ )
+
+
+def test_the_orchestrator_gives_the_loop_a_holder_of_its_own(monkeypatch):
+ """One holder, one contract: the request's summed stats are not the loop's input.
+
+ `stats_holder` accumulates every turn for the reply's usage report, so reading it
+ here would mix turns. The loop gets the per-turn holder instead.
+ """
+ import core.inference.safetensors_agentic as agentic
+ from core.inference.orchestrator import InferenceOrchestrator
+
+ seen = {}
+
+ def capture_loop(**kwargs):
+ seen.update(kwargs)
+ return iter(())
+
+ monkeypatch.setattr(agentic, "run_safetensors_tool_loop", capture_loop)
+ backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
+ backend.active_model_name = "sf-model"
+ backend.models = {"sf-model": {"context_length": 4096}}
+ stats_holder = {"stats": {"usage": {"prompt_tokens": 999}}}
+
+ list(
+ backend.generate_chat_completion_with_tools(
+ messages = list(MESSAGES),
+ tools = [{"type": "function", "function": {"name": "search_conversation"}}],
+ stats_holder = stats_holder,
+ )
+ )
+
+ assert isinstance(seen["generation_stats_holder"], dict)
+ assert seen["generation_stats_holder"] is not stats_holder
+ assert seen["generation_stats_holder"] == {}
+
+
def test_the_loop_omits_the_budget_when_the_context_is_unknown():
"""An absent budget must not become a budget of zero, which would refuse every search."""
seen = {}