Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop charging a chat the whole cache because Max Tokens says "Max" (#10070)

Daniel Han64163939b145
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 43604672c13..1bf103cf51a 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -1703,6 +1703,49 @@ _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS = (
_OPENAI_LLAMA_ADMISSION_IMAGE_EMBEDDING_CAP + _OPENAI_LLAMA_ADMISSION_IMAGE_WRAPPER_TOKENS
)
+# An ESTIMATE where the rest of the sizing is a bound: a run that generates more is
+# undercharged until something re-costs it, which a tool loop does every round boundary and
+# a plain chat cannot yet, so on a full cache a long uncapped generation can still overrun.
+_OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS = (
+ _positive_int_or_none(os.environ.get("UNSLOTH_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS")) or 1024
+)
+
+
+def _openai_llama_admission_context_window(llama_backend) -> Optional[int]:
+ """One slot, which under ``--no-kv-unified`` is 1/N of the aggregate budget."""
+ return _positive_int_or_none(getattr(llama_backend, "context_length", None))
+
+
+def _openai_llama_admission_output_allowance(
+ cap: Optional[int],
+ *,
+ budget: int,
+ prompt_tokens: int,
+ context_window: Optional[int] = None,
+ share: Optional[int] = None,
+) -> int:
+ """KV to reserve for what a request may still generate.
+
+ A cap at or above the window is not a cap: `_build_passthrough_payload` sends
+ max_tokens = backend_ctx and "Max" sends the context length, so both mean unstated, and
+ charging the window for either serialises the queue. Measured against the per-request
+ window, since the budget is N times larger under --no-kv-unified.
+
+ Invariant when a ``share`` is known: an unstated request costs at most its fair share of
+ the cache, so ``capacity`` of them always fit. A flat allowance breaks that on a small
+ cache, where 1024 is most of a share on its own (4096 over four slots admitted three).
+ A prompt already past its share keeps the flat allowance, since it does not fit either
+ way and a zero allowance would only hide that it will still generate.
+ """
+ window = context_window or budget
+ if cap is not None and cap < window:
+ return cap
+ # Clamped to the WINDOW: a request cannot occupy more KV than its own slot holds.
+ allowance = min(_OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS, max(0, window - prompt_tokens))
+ if share is not None and share > prompt_tokens:
+ allowance = min(allowance, share - prompt_tokens)
+ return allowance
+
def _extra_args_image_max_tokens(extra_args) -> Optional[int]:
"""The ``--image-max-tokens N`` a load passed through, or None. Last wins.
@@ -1877,6 +1920,7 @@ def _openai_llama_admission_tokens(
tool_loop: bool = False,
image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,
injected_tools = None,
+ context_window: Optional[int] = None,
) -> Optional[int]:
"""KV a request will occupy: what is sent, plus what it may generate.
@@ -1919,16 +1963,14 @@ def _openai_llama_admission_tokens(
getattr(payload, "max_completion_tokens", None),
)
)
- if cap is None:
- # No cap at all. _build_passthrough_payload then sends max_tokens = backend_ctx,
- # so generation may run until the window is full, and reserving nothing let short
- # uncapped prompts hold tiny commitments while each consumed most of the cache.
- # The honest reservation is the rest of the budget, which does serialise
- # concurrent uncapped requests. That is the true cost of not naming a cap: the
- # alternative is admitting two runs that llama.cpp will then kill.
- output_tokens = max(0, budget - prompt_tokens)
- else:
- output_tokens = cap
+ # Reserving the rest of the budget for either made concurrency 1 for the default chat.
+ output_tokens = _openai_llama_admission_output_allowance(
+ cap,
+ budget = budget,
+ prompt_tokens = prompt_tokens,
+ context_window = context_window,
+ share = max(1, budget // max(1, capacity)),
+ )
# A tool loop opens at its own estimate with an equal share as the floor, and re-costs
# as it grows (generate_chat_completion_with_tools on_conversation_grew ->
# lease.recost_waiting). The floor keeps re-costing rare: under its share a run never
@@ -1987,6 +2029,7 @@ def _openai_llama_admission_reserve(
tool_loop = tool_loop,
image_tokens = _openai_llama_admission_image_tokens(llama_backend),
injected_tools = injected_tools,
+ context_window = _openai_llama_admission_context_window(llama_backend),
)
if payload is not None
else None,
@@ -2052,12 +2095,16 @@ def _openai_llama_admission_recost(
message_image_parts = message_image_parts,
image_tokens = _openai_llama_admission_image_tokens(llama_backend),
)
- if output_tokens is None:
- # No cap named, so generation is bounded only by the window, as the opening
- # estimate assumes. Charging zero here dropped an uncapped loop to its share
- # on round zero while its generations could still fill most of the cache.
- output_tokens = max(0, budget - prompt_tokens)
+ # Reading "Max" literally here would put the run back on the whole cache at its
+ # first round boundary.
share = max(1, budget // max(1, capacity))
+ output_tokens = _openai_llama_admission_output_allowance(
+ output_tokens,
+ budget = budget,
+ prompt_tokens = prompt_tokens,
+ context_window = _openai_llama_admission_context_window(llama_backend),
+ share = share,
+ )
want = max(1, min(budget, max(share, prompt_tokens + max(0, output_tokens))))
lease.recost_waiting(
want,
diff --git a/studio/backend/tests/test_llama_admission_kv_budget.py b/studio/backend/tests/test_llama_admission_kv_budget.py
index b4e18ad29e2..c27a78e8dc9 100644
--- a/studio/backend/tests/test_llama_admission_kv_budget.py
+++ b/studio/backend/tests/test_llama_admission_kv_budget.py
@@ -341,7 +341,9 @@ class TestTheWholeRenderedPromptIsCounted:
An uncapped request reserved no output allowance even though
`_build_passthrough_payload` then sends `max_tokens = backend_ctx`, so short
- prompts held tiny commitments while each generation could fill the cache. And
+ prompts held tiny commitments while each generation could fill the cache. That is
+ now a bounded allowance rather than the whole window, which fixed the undercount by
+ making Studio's default chat un-runnable concurrently. And
the estimate covered only `messages`, while OpenAI tool definitions are
rendered into the prompt and Anthropic keeps `system` and `tools` separate
until they are translated.
@@ -360,18 +362,28 @@ class TestTheWholeRenderedPromptIsCounted:
capacity = capacity,
)
- def test_an_uncapped_request_reserves_the_rest_of_the_window(self):
+ def test_an_uncapped_request_reserves_a_bounded_allowance(self):
+ """It reserved the whole window because generation MAY run that long, which cost
+ the default chat the entire cache before it wrote a token."""
from types import SimpleNamespace
+
+ from routes.inference import _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS
+
payload = SimpleNamespace(
messages = [{"role": "user", "content": "hi"}],
max_tokens = None,
max_completion_tokens = None,
)
- # Generation may run until the window is full, so the reservation says so.
- assert self._cost(payload, budget = 2048) == 2048
+ cost = self._cost(payload, budget = 2048)
+ assert cost < 2048, "an uncapped request still reserves the whole window"
+ assert cost <= _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS + 64
- def test_two_uncapped_short_prompts_do_not_both_run(self):
- """The collision this closes: tiny commitments, cache-filling generations."""
+ def test_uncapped_short_prompts_fill_the_slots_and_no_more(self):
+ """The collision this closes: tiny commitments, cache-filling generations.
+
+ Four fit and a fifth does not, on a cache small enough that the flat allowance would
+ not have left room for four. A prompt-only charge would admit any number.
+ """
from types import SimpleNamespace
async def scenario():
@@ -382,10 +394,11 @@ class TestTheWholeRenderedPromptIsCounted:
max_completion_tokens = None,
)
cost = self._cost(payload, budget = 2048)
- first = await _reserve(queue, capacity = 4, tokens = cost, budget = 2048)
- assert first.lease_nowait() is not None
- second = await _reserve(queue, capacity = 4, tokens = cost, budget = 2048)
- return second.lease_nowait()
+ for _ in range(4):
+ admitted = await _reserve(queue, capacity = 4, tokens = cost, budget = 2048)
+ assert admitted.lease_nowait() is not None
+ fifth = await _reserve(queue, capacity = 4, tokens = cost, budget = 2048)
+ return fifth.lease_nowait()
assert _run(scenario()) is None
diff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
index fa43f551d0e..919b33bd5ca 100644
--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
@@ -549,35 +549,31 @@ class TestARoundIsCostedTheSameWayTheReservationWas:
)
def test_round_zero_keeps_an_uncapped_loop_s_output_allowance(self):
- """No max_tokens and no max_completion_tokens: generation is bounded only by the
- window, which is why the reservation charges ``budget - prompt``. Flattening the
- absent cap to zero drops the loop to its share while its generations can still
- fill most of the cache."""
- from routes.inference import _effective_openai_max_tokens
+ """No max_tokens and no max_completion_tokens.
+
+ The invariant: the round-zero re-cost must not SHRINK the opening lease, because it
+ fires before the conversation has grown, so anything given back is room
+ llama-server is already using. The allowance SIZE is a separate question and it
+ changed, so this asserts the two sides agree rather than asserting a number.
+ """
+ from routes.inference import (
+ _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS,
+ _effective_openai_max_tokens,
+ )
payload = _Payload(
messages = [{"role": "user", "content": "summarise the news"}],
enable_tools = True,
)
assert _effective_openai_max_tokens(payload) is None
- opened, committed, queue = self._round_zero(
+ opened, committed, _queue = self._round_zero(
payload, output_tokens = _effective_openai_max_tokens(payload)
)
- assert opened == 4096, opened
+ assert opened < 4096, f"an uncapped loop still opens on the whole {4096} cache ({opened})"
+ assert opened <= _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS + 64, opened
assert (
committed == opened
), f"round zero shrank an uncapped loop from {opened} to {committed}"
- # And the room it would have released must not admit anyone.
- import asyncio
-
- from core.inference.llama_admission import LlamaAdmissionConfig
-
- async def _newcomer():
- return queue.reserve(
- capacity = 4, config = LlamaAdmissionConfig(), tokens = 2000, budget = 4096
- ).lease_nowait()
-
- assert asyncio.run(_newcomer()) is None
def test_round_zero_keeps_a_top_level_system_prompt(self):
"""Anthropic keeps `system` and `tools` out of `messages` entirely, so for that
diff --git a/studio/backend/tests/test_llama_admission_unstated_max_tokens.py b/studio/backend/tests/test_llama_admission_unstated_max_tokens.py
new file mode 100644
index 00000000000..b6f408fd7e3
--- /dev/null
+++ b/studio/backend/tests/test_llama_admission_unstated_max_tokens.py
@@ -0,0 +1,287 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A cap of the whole window is not a cap, and pricing it as one serialises the queue.
+
+Studio ships "Max Tokens: Max", which sends max_tokens = the context length, and a client
+naming nothing gets the same figure from `_build_passthrough_payload`. #10046 fixed the
+tool-loop half of this; these cover the other half, which is every plain chat.
+"""
+
+from routes.inference import (
+ _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS,
+ _openai_llama_admission_output_allowance,
+ _openai_llama_admission_tokens,
+)
+
+
+class _Payload:
+ def __init__(self, **fields):
+ self.__dict__.update(fields)
+
+ def __getattr__(self, _name):
+ return None
+
+
+def _chat(**fields):
+ return _Payload(messages = [{"role": "user", "content": "hi"}], **fields)
+
+
+class TestWhatCountsAsUnstated:
+ def test_a_real_cap_is_charged_as_asked(self):
+ assert _openai_llama_admission_output_allowance(512, budget = 32768, prompt_tokens = 100) == 512
+
+ def test_no_cap_is_unstated(self):
+ assert (
+ _openai_llama_admission_output_allowance(None, budget = 32768, prompt_tokens = 100)
+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS
+ )
+
+ def test_a_cap_of_the_whole_window_is_unstated(self):
+ """This is the "Max" setting, and it is the default."""
+ assert (
+ _openai_llama_admission_output_allowance(32768, budget = 32768, prompt_tokens = 100)
+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS
+ )
+
+ def test_a_cap_above_the_window_is_unstated(self):
+ assert (
+ _openai_llama_admission_output_allowance(99999, budget = 32768, prompt_tokens = 100)
+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS
+ )
+
+ def test_a_cap_just_under_the_window_is_still_a_cap(self):
+ """Only the whole window reads as unstated; asking for nearly all of it is a
+ statement, and is charged."""
+ assert (
+ _openai_llama_admission_output_allowance(32767, budget = 32768, prompt_tokens = 100)
+ == 32767
+ )
+
+ def test_it_never_exceeds_what_is_left(self):
+ """On a cache smaller than the allowance, stay inside the budget."""
+ assert (
+ _openai_llama_admission_output_allowance(None, budget = 2048, prompt_tokens = 1800) == 248
+ )
+ assert _openai_llama_admission_output_allowance(None, budget = 2048, prompt_tokens = 4000) == 0
+
+
+class TestTheDefaultChatNoLongerTakesTheWholeCache:
+ BUDGET = 32768
+ CAPACITY = 4
+
+ def _cost(self, payload):
+ return _openai_llama_admission_tokens(payload, budget = self.BUDGET, capacity = self.CAPACITY)
+
+ def test_max_tokens_max_does_not_reserve_the_budget(self):
+ cost = self._cost(_chat(max_tokens = self.BUDGET))
+ assert cost < self.BUDGET, "Max Tokens = Max still reserves the whole cache"
+ assert cost <= _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS + 100
+
+ def test_four_default_chats_fit_at_once(self):
+ """The change, as the user sees it: four chats on Max have to fit together."""
+ cost = self._cost(_chat(max_tokens = self.BUDGET))
+ assert cost * 4 <= self.BUDGET, (
+ f"four default chats cost {cost * 4} against a {self.BUDGET} cache, so they "
+ f"cannot decode together"
+ )
+
+ def test_an_uncapped_chat_fits_too(self):
+ cost = self._cost(_chat())
+ assert cost * 4 <= self.BUDGET
+
+ def test_a_named_cap_is_unaffected(self):
+ """Backwards compatible: a stated cap is still priced prompt plus cap."""
+ small = self._cost(_chat(max_tokens = 512))
+ assert small < self._cost(_chat(max_tokens = 8192))
+
+ def test_max_completion_tokens_spelling_counts_too(self):
+ """The supported spelling reaches the same resolver, so "Max" through it counts."""
+ assert self._cost(_chat(max_completion_tokens = self.BUDGET)) < self.BUDGET
+
+
+class TestNothingChangesWhereThereIsNoBudget:
+ def test_an_unknown_budget_still_declines_to_price(self):
+ assert (
+ _openai_llama_admission_tokens(_chat(max_tokens = 4096), budget = None, capacity = 4) is None
+ )
+
+
+class TestMaxIsPerRequestNotPerCache:
+ """Under ``--no-kv-unified`` the budget is the aggregate of N private caches while "Max"
+ is still one slot's context_length, so measuring the cap against the budget reads that
+ default as a real cap and four default chats stop fitting in four caches.
+ """
+
+ WINDOW = 4096 # per slot, and what "Max" sends
+ BUDGET = WINDOW * 4 # four private caches
+
+ def test_max_is_unstated_against_the_per_request_window(self):
+ assert (
+ _openai_llama_admission_output_allowance(
+ self.WINDOW,
+ budget = self.BUDGET,
+ prompt_tokens = 100,
+ context_window = self.WINDOW,
+ )
+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS
+ )
+
+ def test_four_default_chats_fit_four_private_caches(self):
+ cost = _openai_llama_admission_tokens(
+ _chat(max_tokens = self.WINDOW),
+ budget = self.BUDGET,
+ capacity = 4,
+ context_window = self.WINDOW,
+ )
+ assert (
+ cost * 4 <= self.BUDGET
+ ), f"four default chats cost {cost * 4} against {self.BUDGET}, so the fourth waits"
+
+ def test_a_real_cap_under_the_window_is_still_a_cap(self):
+ assert (
+ _openai_llama_admission_output_allowance(
+ 512, budget = self.BUDGET, prompt_tokens = 100, context_window = self.WINDOW
+ )
+ == 512
+ )
+
+ def test_an_unknown_window_falls_back_to_the_budget(self):
+ """Which is the unified case, where the two are the same number anyway."""
+ assert (
+ _openai_llama_admission_output_allowance(
+ self.BUDGET, budget = self.BUDGET, prompt_tokens = 100, context_window = None
+ )
+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS
+ )
+
+ def test_the_window_is_read_off_the_backend(self):
+ from types import SimpleNamespace
+
+ from routes.inference import _openai_llama_admission_context_window
+
+ backend = SimpleNamespace(context_length = self.WINDOW, _kv_cache_context_total = self.BUDGET)
+ assert _openai_llama_admission_context_window(backend) == self.WINDOW
+
+ def test_an_unreadable_window_is_none(self):
+ from types import SimpleNamespace
+
+ from routes.inference import _openai_llama_admission_context_window
+ for value in (None, 0, -1, "nonsense"):
+ assert (
+ _openai_llama_admission_context_window(SimpleNamespace(context_length = value))
+ is None
+ )
+
+
+class TestTheAllowanceFitsTheAdvertisedSlots:
+ """A flat allowance is most of a share on a small cache, so it broke the very thing it
+ was added to fix: at 4096 over four slots a default chat cost 1032 and only three ran,
+ and at 2048 only one did. Clamped to the share, ``capacity`` of them always fit.
+ """
+
+ def _cost(self, budget, capacity):
+ return _openai_llama_admission_tokens(
+ _chat(max_tokens = budget),
+ budget = budget,
+ capacity = capacity,
+ context_window = budget,
+ )
+
+ def test_capacity_default_chats_fit_at_every_cache_size(self):
+ for budget in (2048, 4096, 8192, 32768, 262144):
+ cost = self._cost(budget, 4)
+ assert cost * 4 <= budget, f"{budget} cache admits only {budget // cost} of 4"
+
+ def test_a_large_cache_is_unchanged(self):
+ """The clamp must bite only where the share is the tighter of the two."""
+ assert self._cost(32768, 4) == self._cost(262144, 4)
+
+ def test_the_share_only_ever_lowers_the_allowance(self):
+ base = _openai_llama_admission_output_allowance(
+ None, budget = 4096, prompt_tokens = 100, context_window = 4096
+ )
+ assert (
+ _openai_llama_admission_output_allowance(
+ None, budget = 4096, prompt_tokens = 100, context_window = 4096, share = 1024
+ )
+ < base
+ )
+
+ def test_a_prompt_past_its_share_keeps_the_flat_allowance(self):
+ """Zeroing it would admit a request with no room to generate at all."""
+ assert (
+ _openai_llama_admission_output_allowance(
+ None, budget = 8192, prompt_tokens = 3000, context_window = 8192, share = 2048
+ )
+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS
+ )
+
+ def test_a_named_cap_ignores_the_share(self):
+ assert (
+ _openai_llama_admission_output_allowance(
+ 256, budget = 2048, prompt_tokens = 10, context_window = 2048, share = 512
+ )
+ == 256
+ )
+
+
+class TestTheAllowanceCannotExceedOneSlot:
+ """A request cannot occupy more KV than its own slot holds, so the allowance is clamped
+ against the WINDOW; clamping against the aggregate budget charged a long-prompt chat for
+ more than a slot can physically hold.
+ """
+
+ WINDOW = 4096
+ BUDGET = WINDOW * 4
+
+ def test_a_long_prompt_shrinks_the_allowance_to_fit_its_slot(self):
+ allowance = _openai_llama_admission_output_allowance(
+ self.WINDOW,
+ budget = self.BUDGET,
+ prompt_tokens = 3500,
+ context_window = self.WINDOW,
+ )
+ assert allowance == self.WINDOW - 3500
+ assert 3500 + allowance == self.WINDOW, "a request may not exceed one slot"
+
+ def test_four_long_prompt_chats_still_fit(self):
+ prompt = 3500
+ allowance = _openai_llama_admission_output_allowance(
+ self.WINDOW,
+ budget = self.BUDGET,
+ prompt_tokens = prompt,
+ context_window = self.WINDOW,
+ )
+ assert (prompt + allowance) * 4 <= self.BUDGET
+
+ def test_a_prompt_that_fills_the_slot_gets_nothing(self):
+ assert (
+ _openai_llama_admission_output_allowance(
+ self.WINDOW,
+ budget = self.BUDGET,
+ prompt_tokens = self.WINDOW,
+ context_window = self.WINDOW,
+ )
+ == 0
+ )
+
+ def test_a_short_prompt_still_gets_the_full_allowance(self):
+ assert (
+ _openai_llama_admission_output_allowance(
+ self.WINDOW,
+ budget = self.BUDGET,
+ prompt_tokens = 100,
+ context_window = self.WINDOW,
+ )
+ == _OPENAI_LLAMA_ADMISSION_UNSTATED_OUTPUT_TOKENS
+ )
+
+ def test_unified_is_unchanged(self):
+ """window == budget there, so this clamp is the same arithmetic it always was."""
+ assert (
+ _openai_llama_admission_output_allowance(
+ 2048, budget = 2048, prompt_tokens = 1800, context_window = 2048
+ )
+ == 248
+ )