Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: count rendered GGUF prompts before shared-KV admission (#10794)
Lee Jackson3480890d9228
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 8787d26b27f..c26a7105e78 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -24,6 +24,7 @@ from loggers import get_loggerimport shutilimport signalimport socket+import sslimport subprocessimport sysimport tempfile@@ -6998,6 +6999,12 @@ def _extra_args_have_tensor_split(return bool(str((env or {}).get("LLAMA_ARG_TENSOR_SPLIT", "")).strip())[email protected]_cache(maxsize = 1)+def _count_ssl_context() -> ssl.SSLContext:+ # httpx loads the CA bundle per Client (~15 ms); admission counts once per chat request.+ return ssl.create_default_context()++class LlamaCppBackend:"""Manages a llama-server subprocess for GGUF model inference.@@ -38847,6 +38854,7 @@ class LlamaCppBackend:chat_template_kwargs = None,continue_final_message: bool = False,should_abort = None,+ prefer_native: bool = False,) -> int:"""Count prompt tokens for a chat request via llama-server.@@ -38860,6 +38868,8 @@ class LlamaCppBackend:``should_abort`` is polled between the two llama-server calls. Admission is thecaller's job; this only stops a count that was admitted while idle from spending itssecond round trip once the answer stopped mattering. Raises when it fires.++ ``prefer_native`` tries /v1/chat/completions/input_tokens first (one round trip)."""if not self.is_loaded:if strict:@@ -38908,7 +38918,12 @@ class LlamaCppBackend:tools = neutralize_tool_descriptions(tools, None, _profile)try:- with httpx.Client(timeout = 10, headers = self._auth_headers, trust_env = False) as client:+ with httpx.Client(+ timeout = 10,+ headers = self._auth_headers,+ trust_env = False,+ verify = _count_ssl_context(),+ ) as client:def _tokenize(text: str) -> int:r = client.post(@@ -38955,6 +38970,22 @@ class LlamaCppBackend:if continue_final_message:template_body["continue_final_message"] = Truetemplate_body["add_generation_prompt"] = False+ if prefer_native:+ if should_abort is not None and should_abort():+ raise CountAborted()+ try:+ native = client.post(+ f"{self.base_url}/v1/chat/completions/input_tokens",+ json = template_body,+ )+ if native.status_code == 200:+ count = native.json().get("input_tokens")+ if type(count) is int and count > 0:+ return count+ except Exception:+ pass+ if should_abort is not None and should_abort():+ raise CountAborted()resp = client.post(f"{self.base_url}/apply-template",json = template_body,diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 571b248b73a..5f7856f8a0e 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -2243,6 +2243,7 @@ def _openai_llama_admission_tokens(image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,injected_tools = None,context_window: Optional[int] = None,+ counted_prompt_tokens: Optional[int] = None,vision: bool = True,messages_override = None,) -> Optional[int]:@@ -2264,7 +2265,9 @@ def _openai_llama_admission_tokens(messages = (messages_override if messages_override is not None else getattr(payload, "messages", None))- if isinstance(messages, list) and messages:+ if counted_prompt_tokens is not None:+ prompt_tokens = counted_prompt_tokens+ elif isinstance(messages, list) and messages:try:estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(messages, vision = vision@@ -2372,6 +2375,7 @@ def _openai_llama_admission_reserve(payload = None,tool_loop: bool = False,injected_tools = None,+ counted_prompt_tokens: Optional[int] = None,messages_override = None,tokens = _TOKENS_UNSET,) -> tuple[LlamaAdmissionReservation, LlamaAdmissionConfig]:@@ -2379,7 +2383,16 @@ def _openai_llama_admission_reserve(capacity = _openai_llama_admission_capacity(request, llama_backend)key = str(getattr(llama_backend, "base_url", "llama-server"))budget = _openai_llama_admission_budget(llama_backend)- if tokens is _TOKENS_UNSET:+ if counted_prompt_tokens is not None:+ tokens = _openai_llama_admission_tokens(+ payload,+ counted_prompt_tokens = counted_prompt_tokens,+ budget = budget,+ capacity = capacity,+ tool_loop = tool_loop,+ context_window = _openai_llama_admission_context_window(llama_backend),+ )+ elif tokens is _TOKENS_UNSET:tokens = (_openai_llama_admission_estimate(request = request,@@ -2403,6 +2416,87 @@ def _openai_llama_admission_reserve(return reservation, config+def _count_gguf_admission_prompt(+ llama_backend,+ payload,+ messages,+ tools = None,+ cancel_event = None,+) -> int:+ """Exact prompt count plus media allowance; the whole pool if counting fails+ (the character estimate undercounts numeric text, #10671)."""+ from core.inference.chat_template_helpers import trailing_assistant_text++ budget = _openai_llama_admission_budget(llama_backend) or 0+ try:+ text_messages, images = _openai_llama_admission_messages_for_estimate(+ messages, vision = bool(getattr(llama_backend, "is_vision", False))+ )+ count = llama_backend.count_chat_tokens(+ text_messages,+ tools = tools,+ strict = True,+ prefer_native = images == 0,+ chat_template_kwargs = llama_backend._request_reasoning_kwargs(+ payload.enable_thinking, payload.reasoning_effort, payload.preserve_thinking+ ),+ continue_final_message = _continue_final_message(payload)+ and bool(trailing_assistant_text(messages)),+ **({"should_abort": cancel_event.is_set} if cancel_event is not None else {}),+ )+ if type(count) is not int or count <= 0:+ raise ValueError("Invalid prompt token count")+ return count + _openai_llama_admission_media_tokens(+ payload,+ message_image_parts = images,+ image_tokens = _openai_llama_admission_image_tokens(llama_backend),+ message_video_clips = _conversation_video_clips(messages),+ )+ except Exception:+ logger.debug("Prompt count unavailable; reserving the context pool", exc_info = True)+ return budget+++async def _reserve_counted_gguf_chat(+ *,+ request,+ llama_backend,+ payload,+ messages,+ injected_tools = None,+ tool_loop: bool = False,+):+ config = llama_admission_config_from_env()+ if not (config.enabled and config.kv_budget):+ return await _openai_llama_admission_reserve_async(+ request = request,+ llama_backend = llama_backend,+ payload = payload,+ tool_loop = tool_loop,+ injected_tools = injected_tools,+ )+ identity = (+ getattr(llama_backend, "base_url", None),+ _openai_llama_admission_budget(llama_backend),+ )+ counted = await asyncio.to_thread(+ _count_gguf_admission_prompt, llama_backend, payload, messages, injected_tools+ )+ if identity != (+ getattr(llama_backend, "base_url", None),+ _openai_llama_admission_budget(llama_backend),+ ):+ counted = _openai_llama_admission_budget(llama_backend)+ return _openai_llama_admission_reserve(+ request = request,+ llama_backend = llama_backend,+ payload = payload,+ tool_loop = tool_loop,+ injected_tools = injected_tools,+ counted_prompt_tokens = counted,+ )++async def _openai_llama_admission_reserve_async(*,request: Optional[Request],@@ -2451,6 +2545,7 @@ def _openai_llama_admission_recost(output_tokens: Optional[int] = None,cancel_event = None,injected_tools = None,+ count_prepared_prompt: bool = False,cache_is_empty: bool = False,) -> None:"""Charge a tool loop for what its conversation now is, not what it opened as.@@ -2473,7 +2568,10 @@ def _openai_llama_admission_recost(The gate exists because an idle slot's KV stays resident, which an erased slot's doesnot, so yielding there hands back room that really is free."""- if reservation is None:+ if reservation is None or (cancel_event is not None and cancel_event.is_set()):+ return+ config = llama_admission_config_from_env()+ if not (config.enabled and config.kv_budget):returntry:lease = reservation.lease_nowait()@@ -2483,29 +2581,30 @@ def _openai_llama_admission_recost(if not budget:returncapacity = _openai_llama_admission_capacity(request, llama_backend)- # Every term the OPENING reservation charges, charged again here. Counting fewer- # things than the reservation it replaces would SHRINK a correctly sized lease --- # and since the callback fires at the top of round zero, before any growth, it- # would hand back room llama-server is already using.- estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(- conversation,- vision = bool(getattr(llama_backend, "is_vision", False)),- )- prompt_tokens = estimate_messages_tokens_dense(estimate_messages)- # Re-sent every round, so it belongs in every re-costing, not just the opening one.- prompt_tokens += _openai_llama_admission_injected_tool_tokens(injected_tools)- # Anthropic keeps `system` and `tools` out of the message list entirely, so for- # that route this is most of the prompt.- prompt_tokens += _openai_llama_admission_extra_prompt_tokens(payload)- # mtmd embeddings, KV the message text cannot show: image parts compact to- # "[image]" for the text estimate, so their real cost comes from the compaction- # count. A screenshot tool adds more of them, so this grows with the rounds.- prompt_tokens += _openai_llama_admission_media_tokens(- payload,- message_image_parts = message_image_parts,- image_tokens = _openai_llama_admission_image_tokens(llama_backend),- message_video_clips = _conversation_video_clips(conversation),- )+ if count_prepared_prompt:+ prompt_tokens = _count_gguf_admission_prompt(+ llama_backend, payload, conversation, injected_tools, cancel_event = cancel_event+ )+ if cancel_event is not None and cancel_event.is_set():+ return+ else:+ # Charge every term the opening reservation did: fewer would shrink the lease at round+ # zero and hand back room llama-server is already using.+ estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(+ conversation,+ vision = bool(getattr(llama_backend, "is_vision", False)),+ )+ prompt_tokens = estimate_messages_tokens_dense(estimate_messages)+ prompt_tokens += _openai_llama_admission_injected_tool_tokens(injected_tools)+ # Anthropic keeps `system` and `tools` outside the message list.+ prompt_tokens += _openai_llama_admission_extra_prompt_tokens(payload)+ # mtmd embeddings: image parts compact to "[image]" in the text estimate.+ prompt_tokens += _openai_llama_admission_media_tokens(+ payload,+ message_image_parts = message_image_parts,+ image_tokens = _openai_llama_admission_image_tokens(llama_backend),+ message_video_clips = _conversation_video_clips(conversation),+ )# Reading "Max" literally here would put the run back on the whole cache at its# first round boundary.share = max(1, budget // max(1, capacity))@@ -26456,6 +26555,7 @@ async def produce_openai_chat_completions(# loop down to its share on its very first round.output_tokens = effective_max_tokens,injected_tools = tools_to_use,+ count_prepared_prompt = True,# A round waiting for cache room must still answer Stop. Same event# the loop polls each iteration, so a wait ends where a cancel would.cancel_event = cancel_event,@@ -26538,7 +26638,8 @@ async def produce_openai_chat_completions(_tool_admission_mode = "chat_tool_stream" if payload.stream else "chat_tool_nonstream"try:- reservation, admission_config = await _openai_llama_admission_reserve_async(+ reservation, admission_config = await _reserve_counted_gguf_chat(+ messages = gguf_messages,request = request,llama_backend = llama_backend,payload = payload,@@ -27367,7 +27468,8 @@ async def produce_openai_chat_completions(_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)_tracker.__enter__()try:- reservation, admission_config = await _openai_llama_admission_reserve_async(+ reservation, admission_config = await _reserve_counted_gguf_chat(+ messages = gguf_messages,request = request,llama_backend = llama_backend,payload = payload,@@ -27384,6 +27486,9 @@ async def produce_openai_chat_completions()api_monitor.fail(monitor_id, str(exc))raise _openai_admission_http_exception(exc, status_code = 429)+ except BaseException:+ _tracker.__exit__(None, None, None)+ raiseasync def gguf_stream_chunks():nonlocal _gguf_decode_finished@@ -27711,7 +27816,8 @@ async def produce_openai_chat_completions()else:try:- reservation, admission_config = await _openai_llama_admission_reserve_async(+ reservation, admission_config = await _reserve_counted_gguf_chat(+ messages = gguf_messages,request = request,llama_backend = llama_backend,payload = payload,diff --git a/studio/backend/tests/test_gguf_admission_token_count.py b/studio/backend/tests/test_gguf_admission_token_count.pynew file mode 100644index 00000000000..afc2eeb4f8d--- /dev/null+++ b/studio/backend/tests/test_gguf_admission_token_count.py@@ -0,0 +1,607 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Numeric text must not overcommit the shared KV pool through a character estimate."""++import asyncio+from types import SimpleNamespace+from unittest.mock import Mock++import pytest++from core.inference.llama_admission import LlamaAdmissionQueue+import routes.inference as inference+++def _llama_httpx():+ # Patch the module llama_cpp resolves: another test can swap sys.modules["httpx"].+ from core.inference import llama_cpp+ return llama_cpp.httpx+++def _backend(count):+ return SimpleNamespace(+ base_url = "test-token-count",+ context_length = 30000,+ effective_parallel_slots = 3,+ count_chat_tokens = Mock(return_value = count),+ _request_reasoning_kwargs = Mock(return_value = {"enable_thinking": False}),+ )+++def _payload():+ return SimpleNamespace(+ messages = [{"role": "user", "content": " ".join(map(str, range(1000, 5500)))}],+ max_tokens = 1024,+ enable_thinking = False,+ reasoning_effort = None,+ preserve_thinking = None,+ )+++def test_numeric_prompts_wait_for_room_but_short_prompts_still_overlap(monkeypatch):+ async def scenario(count):+ queue = LlamaAdmissionQueue("counted")+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)+ backend = _backend(count)+ payload = _payload()+ reservations = []+ for _ in range(3):+ reservation, _ = await inference._reserve_counted_gguf_chat(+ request = None,+ llama_backend = backend,+ payload = payload,+ messages = payload.messages,+ )+ reservations.append(reservation)+ leases = [r.lease_nowait() for r in reservations]+ if count == 22560:+ assert leases[0] is not None+ assert leases[1:] == [None, None]+ assert queue.snapshot().committed == 23584+ leases[0].release()+ await asyncio.sleep(0)+ assert reservations[1].lease_nowait() is not None+ else:+ assert all(lease is not None for lease in leases)+ assert backend.count_chat_tokens.call_count == 3++ asyncio.run(scenario(22560))+ asyncio.run(scenario(100))+++def test_counts_prepared_messages_and_tools_with_generation_options():+ backend = _backend(22560)+ payload = _payload()+ prepared = [{"role": "system", "content": "Injected system instructions"}] + payload.messages+ tools = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}]+ count = inference._count_gguf_admission_prompt(backend, payload, prepared, tools)+ assert count == 22560+ args, kwargs = backend.count_chat_tokens.call_args+ assert args[0] == prepared+ assert kwargs == dict(+ tools = tools,+ strict = True,+ prefer_native = True,+ chat_template_kwargs = {"enable_thinking": False},+ continue_final_message = False,+ )+ assert (+ inference._openai_llama_admission_tokens(+ payload,+ budget = 30000,+ capacity = 3,+ injected_tools = tools,+ counted_prompt_tokens = count,+ )+ == 23584+ )++[email protected]("result", [0, -1, None, True, "22560"])+def test_invalid_counts_reserve_the_entire_pool(result):+ payload = _payload()+ assert (+ inference._count_gguf_admission_prompt(_backend(result), payload, payload.messages) == 30000+ )+++def test_tokenizer_failure_reserves_the_entire_pool():+ backend = _backend(0)+ backend.count_chat_tokens.side_effect = RuntimeError("tokenizer unavailable")+ payload = _payload()+ assert inference._count_gguf_admission_prompt(backend, payload, payload.messages) == 30000+++def test_tool_recost_keeps_the_model_count(monkeypatch):+ async def scenario():+ queue = LlamaAdmissionQueue("recost")+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)+ backend = _backend(22560)+ payload = _payload()+ reservation, _ = await inference._reserve_counted_gguf_chat(+ request = None,+ llama_backend = backend,+ payload = payload,+ messages = payload.messages,+ tool_loop = True,+ )+ assert reservation.lease_nowait() is not None+ inference._openai_llama_admission_recost(+ reservation,+ payload.messages,+ request = None,+ llama_backend = backend,+ payload = payload,+ count_prepared_prompt = True,+ )+ assert queue.snapshot().committed == 23584++ asyncio.run(scenario())++[email protected]("failed_path", ["legacy", "native", "unsupported", "invalid", "timeout"])+def test_exact_count_endpoint_failover_preserves_short_chat_concurrency(monkeypatch, failed_path):+ httpx = _llama_httpx()+ from types import MethodType+ from core.inference.llama_cpp import LlamaCppBackend++ calls = []++ def respond(request):+ path = request.url.path+ calls.append(path)+ if path == "/v1/chat/completions/input_tokens":+ if failed_path == "timeout":+ raise httpx.ReadTimeout("count timed out", request = request)+ if failed_path in ("native", "unsupported"):+ return httpx.Response(404 if failed_path == "unsupported" else 503)+ if failed_path == "invalid":+ return httpx.Response(200, json = {"input_tokens": True})+ return httpx.Response(200, json = {"input_tokens": 20})+ if failed_path == "legacy":+ return httpx.Response(503)+ if path == "/apply-template":+ return httpx.Response(200, json = {"prompt": "rendered prompt"})+ assert path == "/tokenize"+ return httpx.Response(200, json = {"tokens": list(range(20))})++ client_type = httpx.Client+ transport = httpx.MockTransport(respond)+ monkeypatch.setattr(httpx, "Client", lambda **kw: client_type(transport = transport, **kw))+ backend = _backend(20)+ backend.base_url = "http://llama.test"+ backend.is_loaded = True+ backend.markup_profile = None+ backend._auth_headers = None+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)+ payload = _payload()+ payload.messages = [{"role": "user", "content": "Hello"}]++ async def scenario():+ queue = LlamaAdmissionQueue("endpoint-failover")+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)+ reservations = await asyncio.gather(+ *(+ inference._reserve_counted_gguf_chat(+ request = None,+ llama_backend = backend,+ payload = payload,+ messages = payload.messages,+ )+ for _ in range(3)+ )+ )+ leases = [reservation.lease_nowait() for reservation, _ in reservations]+ try:+ assert all(lease is not None for lease in leases)+ assert queue.snapshot().committed == 3 * (20 + 1024)+ finally:+ for (reservation, _), lease in zip(reservations, leases):+ if lease is not None:+ lease.release()+ else:+ reservation.cancel()++ asyncio.run(scenario())+ assert calls.count("/v1/chat/completions/input_tokens") == 3+ assert calls.count("/apply-template") == (0 if failed_path == "legacy" else 3)+++def test_native_count_preserves_rendering_options_and_tools(monkeypatch):+ httpx = _llama_httpx()+ import json+ from types import MethodType+ from core.inference.llama_cpp import LlamaCppBackend++ bodies = []++ def respond(request):+ assert request.url.path == "/v1/chat/completions/input_tokens"+ bodies.append(json.loads(request.content))+ return httpx.Response(200, json = {"input_tokens": 22560})++ client_type = httpx.Client+ monkeypatch.setattr(+ httpx, "Client", lambda **kw: client_type(transport = httpx.MockTransport(respond), **kw)+ )+ backend = _backend(22560)+ backend.base_url = "http://llama.test"+ backend.is_loaded = True+ backend.markup_profile = None+ backend._auth_headers = None+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)+ messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "Partial"}]+ tools = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}]+ assert (+ backend.count_chat_tokens(+ messages,+ system = "System instructions",+ tools = tools,+ strict = True,+ chat_template_kwargs = {"enable_thinking": False},+ continue_final_message = True,+ prefer_native = True,+ )+ == 22560+ )+ assert bodies == [+ {+ "messages": [{"role": "system", "content": "System instructions"}] + messages,+ "tools": tools,+ "chat_template_kwargs": {"enable_thinking": False},+ "continue_final_message": True,+ "add_generation_prompt": False,+ }+ ]+++def test_all_count_endpoints_unavailable_still_reserves_pool(monkeypatch):+ httpx = _llama_httpx()+ from types import MethodType+ from core.inference.llama_cpp import LlamaCppBackend++ client_type = httpx.Client+ monkeypatch.setattr(+ httpx,+ "Client",+ lambda **kw: client_type(+ transport = httpx.MockTransport(lambda _: httpx.Response(503)), **kw+ ),+ )+ backend = _backend(0)+ backend.base_url = "http://llama.test"+ backend.is_loaded = True+ backend.markup_profile = None+ backend._auth_headers = None+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)+ payload = _payload()+ assert inference._count_gguf_admission_prompt(backend, payload, payload.messages) == 30000+++def test_media_keeps_existing_allowance_without_native_embedding_count():+ backend = _backend(20)+ payload = _payload()+ payload.messages = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Describe this"},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}},+ ],+ }+ ]+ count = inference._count_gguf_admission_prompt(backend, payload, payload.messages)+ assert count == 20 + inference._openai_llama_admission_image_tokens(backend)+ assert backend.count_chat_tokens.call_args.kwargs["prefer_native"] is False+++def test_counting_recovers_after_a_complete_outage(monkeypatch):+ async def scenario():+ queue = LlamaAdmissionQueue("count-recovery")+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)+ backend = _backend(20)+ payload = _payload()+ payload.messages = [{"role": "user", "content": "Hello"}]+ backend.count_chat_tokens.side_effect = RuntimeError("both count paths unavailable")+ first, _ = await inference._reserve_counted_gguf_chat(+ request = None, llama_backend = backend, payload = payload, messages = payload.messages+ )+ lease = first.lease_nowait()+ assert lease is not None+ assert queue.snapshot().committed == 30000+ lease.release()+ backend.count_chat_tokens.side_effect = None+ reservations = [+ (+ await inference._reserve_counted_gguf_chat(+ request = None, llama_backend = backend, payload = payload, messages = payload.messages+ )+ )[0]+ for _ in range(3)+ ]+ leases = [reservation.lease_nowait() for reservation in reservations]+ try:+ assert all(lease is not None for lease in leases)+ assert queue.snapshot().committed == 3 * (20 + 1024)+ finally:+ for reservation, lease in zip(reservations, leases):+ if lease is not None:+ lease.release()+ else:+ reservation.cancel()++ asyncio.run(scenario())+++def test_exact_count_preserves_video_allowance(monkeypatch):+ backend = _backend(20)+ payload = _payload()+ monkeypatch.setattr(inference, "_conversation_video_clips", lambda _: 2)+ media = Mock(return_value = 4096)+ monkeypatch.setattr(inference, "_openai_llama_admission_media_tokens", media)+ assert inference._count_gguf_admission_prompt(backend, payload, payload.messages) == 4116+ assert media.call_args.kwargs["message_video_clips"] == 2+++def test_prepared_message_parsing_runs_off_the_event_loop(monkeypatch):+ import threading++ loop_thread = threading.get_ident()+ parse = inference._openai_llama_admission_messages_for_estimate+ seen = []++ def checked(*args, **kwargs):+ seen.append(threading.get_ident())+ assert threading.get_ident() != loop_thread+ return parse(*args, **kwargs)++ monkeypatch.setattr(inference, "_openai_llama_admission_messages_for_estimate", checked)++ async def scenario():+ payload = _payload()+ reservation, _ = await inference._reserve_counted_gguf_chat(+ request = None, llama_backend = _backend(20), payload = payload, messages = payload.messages+ )+ reservation.cancel()++ asyncio.run(scenario())+ assert seen and all(thread != loop_thread for thread in seen)++[email protected]("continue_request", [False, True])+def test_tool_recost_rechecks_continuation_after_tool_result(monkeypatch, continue_request):+ async def scenario():+ queue = LlamaAdmissionQueue("continuation-recost")+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)+ backend = _backend(12000)+ payload = _payload()+ payload.continue_final_message = continue_request+ payload.messages = [+ {"role": "user", "content": "Look up the result"},+ {"role": "assistant", "content": "Let me check."},+ ]+ reservation, _ = await inference._reserve_counted_gguf_chat(+ request = None,+ llama_backend = backend,+ payload = payload,+ messages = payload.messages,+ tool_loop = True,+ )+ lease = reservation.lease_nowait()+ assert lease is not None+ try:+ assert (+ backend.count_chat_tokens.call_args.kwargs["continue_final_message"]+ is continue_request+ )+ call = {+ "id": "call_1",+ "type": "function",+ "function": {"name": "lookup", "arguments": "{}"},+ }+ conversation = [+ payload.messages[0],+ {**payload.messages[1], "tool_calls": [call]},+ {"role": "tool", "tool_call_id": "call_1", "content": "Found it"},+ ]+ inference._openai_llama_admission_recost(+ reservation,+ conversation,+ request = None,+ llama_backend = backend,+ payload = payload,+ output_tokens = 1024,+ count_prepared_prompt = True,+ )+ assert backend.count_chat_tokens.call_count == 2+ assert backend.count_chat_tokens.call_args.kwargs["continue_final_message"] is False+ assert queue.snapshot().committed == 13024+ finally:+ lease.release()++ asyncio.run(scenario())+++ "disabled", ["UNSLOTH_LLAMA_ADMISSION_CONTROL", "UNSLOTH_LLAMA_ADMISSION_KV_BUDGET"]+)+def test_disabled_admission_skips_tool_round_count(monkeypatch, disabled):+ monkeypatch.setenv(disabled, "0")++ async def scenario():+ backend = _backend(20)+ payload = _payload()+ reservation, _ = await inference._reserve_counted_gguf_chat(+ request = None,+ llama_backend = backend,+ payload = payload,+ messages = payload.messages,+ )+ lease = reservation.lease_nowait()+ assert lease is not None+ try:+ inference._openai_llama_admission_recost(+ reservation,+ payload.messages,+ request = None,+ llama_backend = backend,+ payload = payload,+ count_prepared_prompt = True,+ )+ backend.count_chat_tokens.assert_not_called()+ finally:+ lease.release()++ asyncio.run(scenario())++[email protected]("cancel_during_count", [False, True])+def test_cancelled_tool_round_does_not_count_or_recost(cancel_during_count):+ import threading++ cancel = threading.Event()+ backend = _backend(20)+ payload = _payload()+ lease = Mock()+ reservation = SimpleNamespace(lease_nowait = lambda: lease)+ if cancel_during_count:++ def count(*args, **kwargs):+ assert kwargs["should_abort"]() is False+ cancel.set()+ assert kwargs["should_abort"]() is True+ return 20++ backend.count_chat_tokens.side_effect = count+ else:+ cancel.set()+ inference._openai_llama_admission_recost(+ reservation,+ payload.messages,+ request = None,+ llama_backend = backend,+ payload = payload,+ cancel_event = cancel,+ count_prepared_prompt = True,+ )+ assert backend.count_chat_tokens.call_count == int(cancel_during_count)+ lease.recost_waiting.assert_not_called()+++def test_stop_after_native_count_failure_skips_fallback(monkeypatch):+ import threading++ httpx = _llama_httpx()+ from types import MethodType+ from core.inference.llama_cpp import LlamaCppBackend++ cancel = threading.Event()+ calls = []++ def respond(request):+ calls.append(request.url.path)+ cancel.set()+ return httpx.Response(503)++ client_type = httpx.Client+ monkeypatch.setattr(+ httpx, "Client", lambda **kw: client_type(transport = httpx.MockTransport(respond), **kw)+ )+ backend = _backend(20)+ backend.base_url = "http://llama.test"+ backend.is_loaded = True+ backend.markup_profile = None+ backend._auth_headers = None+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)+ payload = _payload()+ lease = Mock()+ inference._openai_llama_admission_recost(+ SimpleNamespace(lease_nowait = lambda: lease),+ payload.messages,+ request = None,+ llama_backend = backend,+ payload = payload,+ cancel_event = cancel,+ count_prepared_prompt = True,+ )+ assert calls == ["/v1/chat/completions/input_tokens"]+ lease.recost_waiting.assert_not_called()+++def test_cancel_during_stream_admission_exits_tracker(monkeypatch):+ import threading+ from starlette.requests import Request+ from .llama_backend_double import FakeLlamaCppBackend+ from models.inference import ChatCompletionRequest+ from state import active_generations++ entered = threading.Event()+ release = threading.Event()++ class Backend(FakeLlamaCppBackend):+ context_length = 30000+ base_url = "http://cancel-count-test"+ effective_parallel_slots = 3++ def _request_reasoning_kwargs(self, *args):+ return {}++ def count_chat_tokens(self, *args, **kwargs):+ entered.set()+ assert release.wait(5)+ return 20++ monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: Backend())+ monkeypatch.setattr(inference, "_effective_enable_tools", lambda *a, **kw: False)+ monkeypatch.setattr(inference, "_CANCEL_REGISTRY", {})+ monkeypatch.setattr(active_generations, "_ACTIVE", {})+ payload = ChatCompletionRequest(+ messages = [{"role": "user", "content": "Hello"}],+ stream = True,+ enable_tools = False,+ cancel_id = "count-cancel",+ thread_id = "count-cancel",+ )+ request = Request(+ {+ "type": "http",+ "method": "POST",+ "path": "/chat/completions",+ "headers": [],+ "query_string": b"",+ "state": {},+ }+ )++ async def scenario():+ task = asyncio.create_task(+ inference.produce_openai_chat_completions(+ payload,+ request,+ "test-user",+ cancel_on_disconnect = True,+ )+ )+ try:++ async def wait_for_count():+ while not entered.is_set():+ if task.done():+ await task+ pytest.fail("Count was never entered")+ await asyncio.sleep(0.001)++ await asyncio.wait_for(wait_for_count(), timeout = 2)+ assert active_generations.count() == 1+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ assert not inference._CANCEL_REGISTRY+ assert active_generations.count() == 0+ finally:+ release.set()+ task.cancel()+ await asyncio.gather(task, return_exceptions = True)++ asyncio.run(scenario())diff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.pyindex 6901df1954d..05b38536f7b 100644--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py@@ -575,12 +575,18 @@ class TestAnAudioTurnIsChargedByItsDuration:def generate_chat_completion(self, **_kwargs):yield "ok"- charged, estimate = [], inference_route._openai_llama_admission_estimate+ def count_chat_tokens(self, *_args, **_kwargs):+ return 12++ def _request_reasoning_kwargs(self, *_args):+ return None++ charged, tokens = [], inference_route._openai_llama_admission_tokensmonkeypatch.setattr(inference_route, "get_llama_cpp_backend", _AudioGguf)monkeypatch.setattr(inference_route,- "_openai_llama_admission_estimate",- lambda **kw: charged.append(estimate(**kw)) or charged[-1],+ "_openai_llama_admission_tokens",+ lambda *a, **kw: charged.append(tokens(*a, **kw)) or charged[-1],)monkeypatch.setattr(inference_route, "_decode_audio_mono", lambda _raw: (np.zeros(20 * 16000), 16000)diff --git a/studio/backend/tests/test_mcp_images.py b/studio/backend/tests/test_mcp_images.pyindex b341a283f61..625027012fc 100644--- a/studio/backend/tests/test_mcp_images.py+++ b/studio/backend/tests/test_mcp_images.py@@ -2465,7 +2465,11 @@ def test_the_gguf_admission_estimate_parses_off_the_loop():routes_src = inspect.getsource(inference_route)assert "reservation, admission_config = _openai_llama_admission_reserve(" not in routes_src- assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 7+ assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 4+ assert routes_src.count("await _reserve_counted_gguf_chat(") == 3+ counted = inspect.getsource(inference_route._reserve_counted_gguf_chat)+ assert "await asyncio.to_thread(" in counted+ assert "_count_gguf_admission_prompt" in countedwrapper = inspect.getsource(inference_route._openai_llama_admission_reserve_async)assert "await asyncio.to_thread(\n _openai_llama_admission_estimate," in wrapperassert "_messages_mention_mcp_images(" in wrapper