Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Stop charging a chat the whole cache because Max Tokens says "Max" (#10070)
Daniel Han64163939b145
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 43604672c13..1bf103cf51a 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -1703,6 +1703,49 @@ _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS = (_OPENAI_LLAMA_ADMISSION_IMAGE_EMBEDDING_CAP + _OPENAI_LLAMA_ADMISSION_IMAGE_WRAPPER_TOKENS)+# An ESTIMATE where the rest of the sizing is a bound: a run that generates more is+# undercharged until something re-costs it, which a tool loop does every round boundary and+# a plain chat cannot yet, so on a full cache a long uncapped generation can still overrun.+_OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS = (+ _positive_int_or_none(os.environ.get("UNSLOTH_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS")) or 1024+)+++def _openai_llama_admission_context_window(llama_backend) -> Optional[int]:+ """One slot, which under ``--no-kv-unified`` is 1/N of the aggregate budget."""+ return _positive_int_or_none(getattr(llama_backend, "context_length", None))+++def _openai_llama_admission_output_allowance(+ cap: Optional[int],+ *,+ budget: int,+ prompt_tokens: int,+ context_window: Optional[int] = None,+ share: Optional[int] = None,+) -> int:+ """KV to reserve for what a request may still generate.++ A cap at or above the window is not a cap: `_build_passthrough_payload` sends+ max_tokens = backend_ctx and "Max" sends the context length, so both mean unstated, and+ charging the window for either serialises the queue. Measured against the per-request+ window, since the budget is N times larger under --no-kv-unified.++ Invariant when a ``share`` is known: an unstated request costs at most its fair share of+ the cache, so ``capacity`` of them always fit. A flat allowance breaks that on a small+ cache, where 1024 is most of a share on its own (4096 over four slots admitted three).+ A prompt already past its share keeps the flat allowance, since it does not fit either+ way and a zero allowance would only hide that it will still generate.+ """+ window = context_window or budget+ if cap is not None and cap < window:+ return cap+ # Clamped to the WINDOW: a request cannot occupy more KV than its own slot holds.+ allowance = min(_OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS, max(0, window - prompt_tokens))+ if share is not None and share > prompt_tokens:+ allowance = min(allowance, share - prompt_tokens)+ return allowance+def _extra_args_image_max_tokens(extra_args) -> Optional[int]:"""The ``--image-max-tokens N`` a load passed through, or None. Last wins.@@ -1877,6 +1920,7 @@ def _openai_llama_admission_tokens(tool_loop: bool = False,image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,injected_tools = None,+ context_window: Optional[int] = None,) -> Optional[int]:"""KV a request will occupy: what is sent, plus what it may generate.@@ -1919,16 +1963,14 @@ def _openai_llama_admission_tokens(getattr(payload, "max_completion_tokens", None),))- if cap is None:- # No cap at all. _build_passthrough_payload then sends max_tokens = backend_ctx,- # so generation may run until the window is full, and reserving nothing let short- # uncapped prompts hold tiny commitments while each consumed most of the cache.- # The honest reservation is the rest of the budget, which does serialise- # concurrent uncapped requests. That is the true cost of not naming a cap: the- # alternative is admitting two runs that llama.cpp will then kill.- output_tokens = max(0, budget - prompt_tokens)- else:- output_tokens = cap+ # Reserving the rest of the budget for either made concurrency 1 for the default chat.+ output_tokens = _openai_llama_admission_output_allowance(+ cap,+ budget = budget,+ prompt_tokens = prompt_tokens,+ context_window = context_window,+ share = max(1, budget // max(1, capacity)),+ )# A tool loop opens at its own estimate with an equal share as the floor, and re-costs# as it grows (generate_chat_completion_with_tools on_conversation_grew -># lease.recost_waiting). The floor keeps re-costing rare: under its share a run never@@ -1987,6 +2029,7 @@ def _openai_llama_admission_reserve(tool_loop = tool_loop,image_tokens = _openai_llama_admission_image_tokens(llama_backend),injected_tools = injected_tools,+ context_window = _openai_llama_admission_context_window(llama_backend),)if payload is not Noneelse None,@@ -2052,12 +2095,16 @@ def _openai_llama_admission_recost(message_image_parts = message_image_parts,image_tokens = _openai_llama_admission_image_tokens(llama_backend),)- if output_tokens is None:- # No cap named, so generation is bounded only by the window, as the opening- # estimate assumes. Charging zero here dropped an uncapped loop to its share- # on round zero while its generations could still fill most of the cache.- output_tokens = max(0, budget - prompt_tokens)+ # Reading "Max" literally here would put the run back on the whole cache at its+ # first round boundary.share = max(1, budget // max(1, capacity))+ output_tokens = _openai_llama_admission_output_allowance(+ output_tokens,+ budget = budget,+ prompt_tokens = prompt_tokens,+ context_window = _openai_llama_admission_context_window(llama_backend),+ share = share,+ )want = max(1, min(budget, max(share, prompt_tokens + max(0, output_tokens))))lease.recost_waiting(want,diff --git a/studio/backend/tests/test_llama_admission_kv_budget.py b/studio/backend/tests/test_llama_admission_kv_budget.pyindex b4e18ad29e2..c27a78e8dc9 100644--- a/studio/backend/tests/test_llama_admission_kv_budget.py+++ b/studio/backend/tests/test_llama_admission_kv_budget.py@@ -341,7 +341,9 @@ class TestTheWholeRenderedPromptIsCounted:An uncapped request reserved no output allowance even though`_build_passthrough_payload` then sends `max_tokens = backend_ctx`, so short- prompts held tiny commitments while each generation could fill the cache. And+ prompts held tiny commitments while each generation could fill the cache. That is+ now a bounded allowance rather than the whole window, which fixed the undercount by+ making Studio's default chat un-runnable concurrently. Andthe estimate covered only `messages`, while OpenAI tool definitions arerendered into the prompt and Anthropic keeps `system` and `tools` separateuntil they are translated.@@ -360,18 +362,28 @@ class TestTheWholeRenderedPromptIsCounted:capacity = capacity,)- def test_an_uncapped_request_reserves_the_rest_of_the_window(self):+ def test_an_uncapped_request_reserves_a_bounded_allowance(self):+ """It reserved the whole window because generation MAY run that long, which cost+ the default chat the entire cache before it wrote a token."""from types import SimpleNamespace++ from routes.inference import _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS+payload = SimpleNamespace(messages = [{"role": "user", "content": "hi"}],max_tokens = None,max_completion_tokens = None,)- # Generation may run until the window is full, so the reservation says so.- assert self._cost(payload, budget = 2048) == 2048+ cost = self._cost(payload, budget = 2048)+ assert cost < 2048, "an uncapped request still reserves the whole window"+ assert cost <= _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS + 64- def test_two_uncapped_short_prompts_do_not_both_run(self):- """The collision this closes: tiny commitments, cache-filling generations."""+ def test_uncapped_short_prompts_fill_the_slots_and_no_more(self):+ """The collision this closes: tiny commitments, cache-filling generations.++ Four fit and a fifth does not, on a cache small enough that the flat allowance would+ not have left room for four. A prompt-only charge would admit any number.+ """from types import SimpleNamespaceasync def scenario():@@ -382,10 +394,11 @@ class TestTheWholeRenderedPromptIsCounted:max_completion_tokens = None,)cost = self._cost(payload, budget = 2048)- first = await _reserve(queue, capacity = 4, tokens = cost, budget = 2048)- assert first.lease_nowait() is not None- second = await _reserve(queue, capacity = 4, tokens = cost, budget = 2048)- return second.lease_nowait()+ for _ in range(4):+ admitted = await _reserve(queue, capacity = 4, tokens = cost, budget = 2048)+ assert admitted.lease_nowait() is not None+ fifth = await _reserve(queue, capacity = 4, tokens = cost, budget = 2048)+ return fifth.lease_nowait()assert _run(scenario()) is Nonediff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.pyindex fa43f551d0e..919b33bd5ca 100644--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py@@ -549,35 +549,31 @@ class TestARoundIsCostedTheSameWayTheReservationWas:)def test_round_zero_keeps_an_uncapped_loop_s_output_allowance(self):- """No max_tokens and no max_completion_tokens: generation is bounded only by the- window, which is why the reservation charges ``budget - prompt``. Flattening the- absent cap to zero drops the loop to its share while its generations can still- fill most of the cache."""- from routes.inference import _effective_openai_max_tokens+ """No max_tokens and no max_completion_tokens.++ The invariant: the round-zero re-cost must not SHRINK the opening lease, because it+ fires before the conversation has grown, so anything given back is room+ llama-server is already using. The allowance SIZE is a separate question and it+ changed, so this asserts the two sides agree rather than asserting a number.+ """+ from routes.inference import (+ _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS,+ _effective_openai_max_tokens,+ )payload = _Payload(messages = [{"role": "user", "content": "summarise the news"}],enable_tools = True,)assert _effective_openai_max_tokens(payload) is None- opened, committed, queue = self._round_zero(+ opened, committed, _queue = self._round_zero(payload, output_tokens = _effective_openai_max_tokens(payload))- assert opened == 4096, opened+ assert opened < 4096, f"an uncapped loop still opens on the whole {4096} cache ({opened})"+ assert opened <= _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS + 64, openedassert (committed == opened), f"round zero shrank an uncapped loop from {opened} to {committed}"- # And the room it would have released must not admit anyone.- import asyncio-- from core.inference.llama_admission import LlamaAdmissionConfig-- async def _newcomer():- return queue.reserve(- capacity = 4, config = LlamaAdmissionConfig(), tokens = 2000, budget = 4096- ).lease_nowait()-- assert asyncio.run(_newcomer()) is Nonedef test_round_zero_keeps_a_top_level_system_prompt(self):"""Anthropic keeps `system` and `tools` out of `messages` entirely, so for thatdiff --git a/studio/backend/tests/test_llama_admission_unstated_max_tokens.py b/studio/backend/tests/test_llama_admission_unstated_max_tokens.pynew file mode 100644index 00000000000..b6f408fd7e3--- /dev/null+++ b/studio/backend/tests/test_llama_admission_unstated_max_tokens.py@@ -0,0 +1,287 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A cap of the whole window is not a cap, and pricing it as one serialises the queue.++Studio ships "Max Tokens: Max", which sends max_tokens = the context length, and a client+naming nothing gets the same figure from `_build_passthrough_payload`. #10046 fixed the+tool-loop half of this; these cover the other half, which is every plain chat.+"""++from routes.inference import (+ _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS,+ _openai_llama_admission_output_allowance,+ _openai_llama_admission_tokens,+)+++class _Payload:+ def __init__(self, **fields):+ self.__dict__.update(fields)++ def __getattr__(self, _name):+ return None+++def _chat(**fields):+ return _Payload(messages = [{"role": "user", "content": "hi"}], **fields)+++class TestWhatCountsAsUnstated:+ def test_a_real_cap_is_charged_as_asked(self):+ assert _openai_llama_admission_output_allowance(512, budget = 32768, prompt_tokens = 100) == 512++ def test_no_cap_is_unstated(self):+ assert (+ _openai_llama_admission_output_allowance(None, budget = 32768, prompt_tokens = 100)+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS+ )++ def test_a_cap_of_the_whole_window_is_unstated(self):+ """This is the "Max" setting, and it is the default."""+ assert (+ _openai_llama_admission_output_allowance(32768, budget = 32768, prompt_tokens = 100)+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS+ )++ def test_a_cap_above_the_window_is_unstated(self):+ assert (+ _openai_llama_admission_output_allowance(99999, budget = 32768, prompt_tokens = 100)+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS+ )++ def test_a_cap_just_under_the_window_is_still_a_cap(self):+ """Only the whole window reads as unstated; asking for nearly all of it is a+ statement, and is charged."""+ assert (+ _openai_llama_admission_output_allowance(32767, budget = 32768, prompt_tokens = 100)+ == 32767+ )++ def test_it_never_exceeds_what_is_left(self):+ """On a cache smaller than the allowance, stay inside the budget."""+ assert (+ _openai_llama_admission_output_allowance(None, budget = 2048, prompt_tokens = 1800) == 248+ )+ assert _openai_llama_admission_output_allowance(None, budget = 2048, prompt_tokens = 4000) == 0+++class TestTheDefaultChatNoLongerTakesTheWholeCache:+ BUDGET = 32768+ CAPACITY = 4++ def _cost(self, payload):+ return _openai_llama_admission_tokens(payload, budget = self.BUDGET, capacity = self.CAPACITY)++ def test_max_tokens_max_does_not_reserve_the_budget(self):+ cost = self._cost(_chat(max_tokens = self.BUDGET))+ assert cost < self.BUDGET, "Max Tokens = Max still reserves the whole cache"+ assert cost <= _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS + 100++ def test_four_default_chats_fit_at_once(self):+ """The change, as the user sees it: four chats on Max have to fit together."""+ cost = self._cost(_chat(max_tokens = self.BUDGET))+ assert cost * 4 <= self.BUDGET, (+ f"four default chats cost {cost * 4} against a {self.BUDGET} cache, so they "+ f"cannot decode together"+ )++ def test_an_uncapped_chat_fits_too(self):+ cost = self._cost(_chat())+ assert cost * 4 <= self.BUDGET++ def test_a_named_cap_is_unaffected(self):+ """Backwards compatible: a stated cap is still priced prompt plus cap."""+ small = self._cost(_chat(max_tokens = 512))+ assert small < self._cost(_chat(max_tokens = 8192))++ def test_max_completion_tokens_spelling_counts_too(self):+ """The supported spelling reaches the same resolver, so "Max" through it counts."""+ assert self._cost(_chat(max_completion_tokens = self.BUDGET)) < self.BUDGET+++class TestNothingChangesWhereThereIsNoBudget:+ def test_an_unknown_budget_still_declines_to_price(self):+ assert (+ _openai_llama_admission_tokens(_chat(max_tokens = 4096), budget = None, capacity = 4) is None+ )+++class TestMaxIsPerRequestNotPerCache:+ """Under ``--no-kv-unified`` the budget is the aggregate of N private caches while "Max"+ is still one slot's context_length, so measuring the cap against the budget reads that+ default as a real cap and four default chats stop fitting in four caches.+ """++ WINDOW = 4096 # per slot, and what "Max" sends+ BUDGET = WINDOW * 4 # four private caches++ def test_max_is_unstated_against_the_per_request_window(self):+ assert (+ _openai_llama_admission_output_allowance(+ self.WINDOW,+ budget = self.BUDGET,+ prompt_tokens = 100,+ context_window = self.WINDOW,+ )+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS+ )++ def test_four_default_chats_fit_four_private_caches(self):+ cost = _openai_llama_admission_tokens(+ _chat(max_tokens = self.WINDOW),+ budget = self.BUDGET,+ capacity = 4,+ context_window = self.WINDOW,+ )+ assert (+ cost * 4 <= self.BUDGET+ ), f"four default chats cost {cost * 4} against {self.BUDGET}, so the fourth waits"++ def test_a_real_cap_under_the_window_is_still_a_cap(self):+ assert (+ _openai_llama_admission_output_allowance(+ 512, budget = self.BUDGET, prompt_tokens = 100, context_window = self.WINDOW+ )+ == 512+ )++ def test_an_unknown_window_falls_back_to_the_budget(self):+ """Which is the unified case, where the two are the same number anyway."""+ assert (+ _openai_llama_admission_output_allowance(+ self.BUDGET, budget = self.BUDGET, prompt_tokens = 100, context_window = None+ )+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS+ )++ def test_the_window_is_read_off_the_backend(self):+ from types import SimpleNamespace++ from routes.inference import _openai_llama_admission_context_window++ backend = SimpleNamespace(context_length = self.WINDOW, _kv_cache_context_total = self.BUDGET)+ assert _openai_llama_admission_context_window(backend) == self.WINDOW++ def test_an_unreadable_window_is_none(self):+ from types import SimpleNamespace++ from routes.inference import _openai_llama_admission_context_window+ for value in (None, 0, -1, "nonsense"):+ assert (+ _openai_llama_admission_context_window(SimpleNamespace(context_length = value))+ is None+ )+++class TestTheAllowanceFitsTheAdvertisedSlots:+ """A flat allowance is most of a share on a small cache, so it broke the very thing it+ was added to fix: at 4096 over four slots a default chat cost 1032 and only three ran,+ and at 2048 only one did. Clamped to the share, ``capacity`` of them always fit.+ """++ def _cost(self, budget, capacity):+ return _openai_llama_admission_tokens(+ _chat(max_tokens = budget),+ budget = budget,+ capacity = capacity,+ context_window = budget,+ )++ def test_capacity_default_chats_fit_at_every_cache_size(self):+ for budget in (2048, 4096, 8192, 32768, 262144):+ cost = self._cost(budget, 4)+ assert cost * 4 <= budget, f"{budget} cache admits only {budget // cost} of 4"++ def test_a_large_cache_is_unchanged(self):+ """The clamp must bite only where the share is the tighter of the two."""+ assert self._cost(32768, 4) == self._cost(262144, 4)++ def test_the_share_only_ever_lowers_the_allowance(self):+ base = _openai_llama_admission_output_allowance(+ None, budget = 4096, prompt_tokens = 100, context_window = 4096+ )+ assert (+ _openai_llama_admission_output_allowance(+ None, budget = 4096, prompt_tokens = 100, context_window = 4096, share = 1024+ )+ < base+ )++ def test_a_prompt_past_its_share_keeps_the_flat_allowance(self):+ """Zeroing it would admit a request with no room to generate at all."""+ assert (+ _openai_llama_admission_output_allowance(+ None, budget = 8192, prompt_tokens = 3000, context_window = 8192, share = 2048+ )+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS+ )++ def test_a_named_cap_ignores_the_share(self):+ assert (+ _openai_llama_admission_output_allowance(+ 256, budget = 2048, prompt_tokens = 10, context_window = 2048, share = 512+ )+ == 256+ )+++class TestTheAllowanceCannotExceedOneSlot:+ """A request cannot occupy more KV than its own slot holds, so the allowance is clamped+ against the WINDOW; clamping against the aggregate budget charged a long-prompt chat for+ more than a slot can physically hold.+ """++ WINDOW = 4096+ BUDGET = WINDOW * 4++ def test_a_long_prompt_shrinks_the_allowance_to_fit_its_slot(self):+ allowance = _openai_llama_admission_output_allowance(+ self.WINDOW,+ budget = self.BUDGET,+ prompt_tokens = 3500,+ context_window = self.WINDOW,+ )+ assert allowance == self.WINDOW - 3500+ assert 3500 + allowance == self.WINDOW, "a request may not exceed one slot"++ def test_four_long_prompt_chats_still_fit(self):+ prompt = 3500+ allowance = _openai_llama_admission_output_allowance(+ self.WINDOW,+ budget = self.BUDGET,+ prompt_tokens = prompt,+ context_window = self.WINDOW,+ )+ assert (prompt + allowance) * 4 <= self.BUDGET++ def test_a_prompt_that_fills_the_slot_gets_nothing(self):+ assert (+ _openai_llama_admission_output_allowance(+ self.WINDOW,+ budget = self.BUDGET,+ prompt_tokens = self.WINDOW,+ context_window = self.WINDOW,+ )+ == 0+ )++ def test_a_short_prompt_still_gets_the_full_allowance(self):+ assert (+ _openai_llama_admission_output_allowance(+ self.WINDOW,+ budget = self.BUDGET,+ prompt_tokens = 100,+ context_window = self.WINDOW,+ )+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS+ )++ def test_unified_is_unchanged(self):+ """window == budget there, so this clamp is the same arithmetic it always was."""+ assert (+ _openai_llama_admission_output_allowance(+ 2048, budget = 2048, prompt_tokens = 1800, context_window = 2048+ )+ == 248+ )