Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
studio: use generation usage for conversation search budgets (#9438)
Maheswar Kumar6e21a68ebdfe
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 72bf5884fde..c3ffe241dad 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -973,30 +973,39 @@ class InferenceBackend:from core.inference.safetensors_agentic import run_safetensors_tool_loopfrom core.inference.tools import execute_tool+ # The usage of the LATEST turn, so the loop can size a search against a real+ # token count. Cleared before every turn and written when that turn ends, so a+ # turn that failed or reported nothing leaves it empty rather than stale.+ _turn_stats: dict = {}+def _single_turn(conv: list, *, active_tools: Optional[list[dict]] = None):# conv already has the system message -- avoid double-prepend.# `active_tools` is supplied by run_safetensors_tool_loop so one-shot# tools such as render_html can be removed from later same-response prompts.turn_tools = active_tools if active_tools is not None else tools- yield from self._generate_chat_response_inner(- messages = conv,- system_prompt = "",- temperature = temperature,- top_p = top_p,- top_k = top_k,- min_p = min_p,- max_new_tokens = max_new_tokens,- repetition_penalty = repetition_penalty,- cancel_event = cancel_event,- tools = turn_tools,- enable_thinking = enable_thinking,- reasoning_effort = reasoning_effort,- preserve_thinking = preserve_thinking,- # Self-limiting: after a tool call the conversation ends on a tool- # result, so later turns render as ordinary new turns.- continue_final_message = continue_final_message,- presence_penalty = presence_penalty,- )+ _turn_stats["stats"] = None+ try:+ yield from self._generate_chat_response_inner(+ messages = conv,+ system_prompt = "",+ temperature = temperature,+ top_p = top_p,+ top_k = top_k,+ min_p = min_p,+ max_new_tokens = max_new_tokens,+ repetition_penalty = repetition_penalty,+ cancel_event = cancel_event,+ tools = turn_tools,+ enable_thinking = enable_thinking,+ reasoning_effort = reasoning_effort,+ preserve_thinking = preserve_thinking,+ # Self-limiting: after a tool call the conversation ends on a tool+ # result, so later turns render as ordinary new turns.+ continue_final_message = continue_final_message,+ presence_penalty = presence_penalty,+ )+ finally:+ _turn_stats["stats"] = self.last_generation_statsinitial = list(messages)if system_prompt:@@ -1042,6 +1051,7 @@ class InferenceBackend:# So a conversation search can be sized against what this model can hold.context_length = _model_info.get("context_length"),max_tokens = max_new_tokens,+ generation_stats_holder = _turn_stats,)def generate_chat_response(diff --git a/studio/backend/core/inference/orchestrator.py b/studio/backend/core/inference/orchestrator.pyindex d1e3c27a2a5..a6b5f93833d 100644--- a/studio/backend/core/inference/orchestrator.py+++ b/studio/backend/core/inference/orchestrator.py@@ -2206,12 +2206,18 @@ class InferenceOrchestrator:max_new_tokens = max_tokens if max_tokens and max_tokens > 0 else 2048+ # The worker's usage for the LATEST turn only. Hoisted out of the turn so the+ # loop can size a conversation search against a real prompt count, and cleared+ # on the way in rather than on each way out, so a turn that failed or was+ # cancelled leaves it empty instead of handing the loop an earlier turn's number.+ turn_stats: dict = {}+def _single_turn(conv: list, *, active_tools: Optional[list[dict]] = None):# ``conv`` already carries any system message. ``active_tools`` lets# run_safetensors_tool_loop drop one-shot tools (e.g. render_html) from# later same-response prompts.turn_tools = active_tools if active_tools is not None else tools- turn_stats: dict = {}+ turn_stats.clear()common_kwargs = dict(messages = conv,system_prompt = "",@@ -2315,6 +2321,7 @@ class InferenceOrchestrator:# So a conversation search can be sized against what this model can hold.context_length = _model_info.get("context_length"),max_tokens = max_new_tokens,+ generation_stats_holder = turn_stats,)def generate_with_adapter_control(diff --git a/studio/backend/core/inference/safetensors_agentic.py b/studio/backend/core/inference/safetensors_agentic.pyindex 8eb31e1afa4..97be8397b5b 100644--- a/studio/backend/core/inference/safetensors_agentic.py+++ b/studio/backend/core/inference/safetensors_agentic.py@@ -470,6 +470,47 @@ def _call_single_turn(single_turn, conversation: list, active_tools: list[dict])return single_turn(conversation)+def _dense_message_tokens(messages: list[dict]) -> int:+ """`estimate_messages_tokens_dense`, imported where it is used like the rest here."""+ from core.inference.context_window import estimate_messages_tokens_dense+ return estimate_messages_tokens_dense(messages)+++def _spent_prompt_tokens(+ conversation: list[dict],+ tools: Optional[list[dict]],+ generation_stats_holder: Optional[dict],+ prompt_dense_tokens: int,+) -> int:+ """Tokens the next prompt already owes, from the count the last turn reported.++ The backend tokenises the turn's prompt to run it and ships that count on gen_done,+ tool catalogue included, so the only part left to estimate is what the loop appended+ afterwards: this turn's assistant text and the results of any tool already run in the+ same batch. Estimating that tail alone is what stops a long English preamble being+ charged several times what it costs.++ An exact recount here is not available. This loop runs in the PARENT process and+ `InferenceOrchestrator.models` mirrors the worker's model_info, which carries no+ tokenizer, so counting the conversation again would mean a round trip into the worker+ between every tool call.++ Without a report the estimate covers the whole thread, as it did before. Dense+ because four characters per token undercounts CJK and emoji by about half: measured+ on an 81-message CJK chat, 1295 estimated against 2737 real, reporting 1777 tokens of+ room where 335 remained. Never floored to zero, which reaches the tool as "there is+ no room left in this context to search earlier conversation" and switches recall off+ on exactly the tight windows that need it.+ """+ stats = (generation_stats_holder or {}).get("stats")+ usage = stats.get("usage") if isinstance(stats, dict) else None+ prompt_tokens = usage.get("prompt_tokens") if isinstance(usage, dict) else None+ if isinstance(prompt_tokens, int) and not isinstance(prompt_tokens, bool) and prompt_tokens > 0:+ added = _dense_message_tokens(conversation) - prompt_dense_tokens+ return prompt_tokens + max(0, added)+ return _dense_message_tokens(conversation) + _dense_message_tokens(tools or [])++def run_safetensors_tool_loop(*,single_turn: Callable[[list], Generator[str, None, None]],@@ -493,6 +534,7 @@ def run_safetensors_tool_loop(renderable_tools = None,context_length: Optional[int] = None,max_tokens: Optional[int] = None,+ generation_stats_holder: Optional[dict] = None,) -> Generator[dict, None, None]:"""Drive an agentic tool loop on top of a cumulative-text generator.@@ -698,6 +740,9 @@ def run_safetensors_tool_loop(return Falsereturn _first_detected_tool_name(content) == "render_html"+ # The conversation as this turn's prompt renders it, so what the loop appends+ # afterwards can be charged on its own against the count the turn reports.+ prompt_dense_tokens = _dense_message_tokens(conversation)gen = _call_single_turn(single_turn, conversation, active_tools)prev_cumulative = ""@@ -1325,26 +1370,27 @@ def run_safetensors_tool_loop(if context_length and _accepts_kwarg(execute_tool, "conversation_budget_tokens"):- from core.inference.context_window import (- estimate_messages_tokens_dense,- retrieval_budget,- )+ from core.inference.context_window import retrieval_budget- # Dense, unlike the eviction estimator: four characters per token- # undercounts CJK and emoji by about half, and this path has no- # rolling fit to recover if the tool exchange it sizes then puts- # the next prompt over the window. Measured on an 81-message CJK- # chat: 1295 estimated against 2737 real, reporting 1777 tokens of- # room where 335 remained.+ # From the tokenizer count the last turn reported rather than+ # from characters alone: `conversation` already carries this+ # turn's assistant preamble and its tool call, and pricing the+ # whole thread by characters is what hands a retrieval room the+ # next prompt does not have.## `reply_returns`, as the GGUF loop does: result and reply are# both protected on the next fit, so one retrieval cannot spend# the budget they share.+ spent = _spent_prompt_tokens(+ conversation,+ tools,+ generation_stats_holder,+ prompt_dense_tokens,+ )kwargs["conversation_budget_tokens"] = retrieval_budget(int(context_length),max_tokens,- estimate_messages_tokens_dense(conversation)- + estimate_messages_tokens_dense(tools or []),+ spent,reply_returns = True,)# And what a RESULT may add, which is the same question asked ofdiff --git a/studio/backend/tests/test_conversation_search_safetensors_loop.py b/studio/backend/tests/test_conversation_search_safetensors_loop.pyindex f9316877f9f..3f8d7e7a641 100644--- a/studio/backend/tests/test_conversation_search_safetensors_loop.py+++ b/studio/backend/tests/test_conversation_search_safetensors_loop.py@@ -116,9 +116,7 @@ def test_the_budget_charges_token_dense_text_at_its_real_rate():single_turn = _searching_model(1),messages = list(dense_messages),tools = tools,- execute_tool = lambda name, arguments, **kwargs: (- seen.update(kwargs) or "an earlier turn"- ),+ execute_tool = lambda name, arguments, **kwargs: seen.update(kwargs) or "an earlier turn",cancel_event = threading.Event(),max_tool_iterations = 2,thread_id = "t-sf",@@ -180,6 +178,154 @@ def test_the_loop_budgets_its_search_against_this_models_context():assert budget < prompt_budget(4096, 512) - estimate_messages_tokens(MESSAGES)+def _model_that_thinks_out_loud(preamble, stats_holder, reported):+ """A model that writes prose before its one search, reporting usage as a turn does."""+ state = {"n": 0}++ def single_turn(messages):+ stats_holder["stats"] = None+ state["n"] += 1+ if state["n"] > 1:+ yield "done"+ return+ call = (+ preamble+ + '<tool_call>{"name":"search_conversation","arguments":{"query":"the code"}}'+ + "</tool_call>"+ )+ buffer = ""+ for char in call:+ buffer += char+ yield buffer+ stats_holder["stats"] = {"usage": {"prompt_tokens": reported}}++ return single_turn+++def _budget_after_a_preamble(preamble, reported, stats_holder):+ seen = {}+ list(+ run_safetensors_tool_loop(+ single_turn = _model_that_thinks_out_loud(preamble, stats_holder, reported),+ messages = [+ {"role": "user", "content": "recall the deployment steps. " * 420},+ {"role": "user", "content": "what was the code from earlier"},+ ],+ tools = [{"type": "function", "function": {"name": "search_conversation"}}],+ execute_tool = lambda name, arguments, **kwargs: seen.update(kwargs) or "an earlier turn",+ cancel_event = threading.Event(),+ max_tool_iterations = 2,+ thread_id = "t-sf",+ context_length = 8192,+ max_tokens = 512,+ generation_stats_holder = stats_holder,+ )+ )+ return seen["conversation_budget_tokens"]+++def test_the_budget_charges_this_turns_own_output_in_tokens_not_bytes():+ """A 4600-character English preamble costs about 1150 tokens, not 4600.++ Charged by its byte length it takes the whole prompt budget of an 8K window, so the+ search is refused with roughly 3.4K tokens still free, which is worse than the+ estimate it replaces. The fixture is deliberately large: on a three-line chat the+ two numbers differ by too little to fail an assertion.+ """+ from core.inference.context_window import retrieval_budget++ preamble = "the answer is somewhere in the earlier turns. " * 100+ budget = _budget_after_a_preamble(preamble, 3_067, {})++ # Charged as its 4600 bytes this is nothing. Charged as tokens it is a real search.+ assert budget > 3_000+ # And it IS charged: the room is smaller than it was before the turn wrote anything.+ assert budget < retrieval_budget(8192, 512, 3_067, reply_returns = True)+++def test_the_budget_spends_the_count_the_turn_reported():+ """The reported prompt count is a tokenizer's, so it outranks the character estimate.++ Six thousand tokens of a window this size leaves room for a small retrieval and no+ more; the estimate alone would price the same thread at a third of that and hand out+ room the next prompt does not have.+ """+ budget = _budget_after_a_preamble("thinking. ", 6_000, {})++ assert 0 < budget < 1_800+++def test_the_budget_estimates_when_no_usage_was_reported():+ """A cancelled or unreported turn falls back to the estimate, never to zero.++ Zero reaches the tool as "there is no room left in this context to search earlier+ conversation", the same failure `_recall_top_k` carries a note about: it switches+ recall off on exactly the tight windows that need it.+ """+ from core.inference.context_window import (+ estimate_messages_tokens_dense,+ retrieval_budget,+ )++ seen = {}+ tools = [{"type": "function", "function": {"name": "search_conversation"}}]++ list(+ run_safetensors_tool_loop(+ single_turn = _searching_model(1),+ messages = list(MESSAGES),+ tools = tools,+ execute_tool = lambda name, arguments, **kwargs: seen.update(kwargs) or "an earlier turn",+ cancel_event = threading.Event(),+ max_tool_iterations = 2,+ thread_id = "t-sf",+ context_length = 4096,+ max_tokens = 512,+ generation_stats_holder = {"stats": None},+ )+ )++ budget = seen["conversation_budget_tokens"]+ assert budget > 0+ assert budget <= retrieval_budget(+ 4096, 512, estimate_messages_tokens_dense(tools), reply_returns = True+ )+++def test_the_orchestrator_gives_the_loop_a_holder_of_its_own(monkeypatch):+ """One holder, one contract: the request's summed stats are not the loop's input.++ `stats_holder` accumulates every turn for the reply's usage report, so reading it+ here would mix turns. The loop gets the per-turn holder instead.+ """+ import core.inference.safetensors_agentic as agentic+ from core.inference.orchestrator import InferenceOrchestrator++ seen = {}++ def capture_loop(**kwargs):+ seen.update(kwargs)+ return iter(())++ monkeypatch.setattr(agentic, "run_safetensors_tool_loop", capture_loop)+ backend = InferenceOrchestrator.__new__(InferenceOrchestrator)+ backend.active_model_name = "sf-model"+ backend.models = {"sf-model": {"context_length": 4096}}+ stats_holder = {"stats": {"usage": {"prompt_tokens": 999}}}++ list(+ backend.generate_chat_completion_with_tools(+ messages = list(MESSAGES),+ tools = [{"type": "function", "function": {"name": "search_conversation"}}],+ stats_holder = stats_holder,+ )+ )++ assert isinstance(seen["generation_stats_holder"], dict)+ assert seen["generation_stats_holder"] is not stats_holder+ assert seen["generation_stats_holder"] == {}++def test_the_loop_omits_the_budget_when_the_context_is_unknown():"""An absent budget must not become a budget of zero, which would refuse every search."""seen = {}