Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: count rendered GGUF prompts before shared-KV admission (#10794)

Lee Jackson3480890d9228
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 8787d26b27f..c26a7105e78 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -24,6 +24,7 @@ from loggers import get_logger
import shutil
import signal
import socket
+import ssl
import subprocess
import sys
import tempfile
@@ -6998,6 +6999,12 @@ def _extra_args_have_tensor_split(
return bool(str((env or {}).get("LLAMA_ARG_TENSOR_SPLIT", "")).strip())
[email protected]_cache(maxsize = 1)
+def _count_ssl_context() -> ssl.SSLContext:
+ # httpx loads the CA bundle per Client (~15 ms); admission counts once per chat request.
+ return ssl.create_default_context()
+
+
class LlamaCppBackend:
"""Manages a llama-server subprocess for GGUF model inference.
@@ -38847,6 +38854,7 @@ class LlamaCppBackend:
chat_template_kwargs = None,
continue_final_message: bool = False,
should_abort = None,
+ prefer_native: bool = False,
) -> int:
"""Count prompt tokens for a chat request via llama-server.
@@ -38860,6 +38868,8 @@ class LlamaCppBackend:
``should_abort`` is polled between the two llama-server calls. Admission is the
caller's job; this only stops a count that was admitted while idle from spending its
second round trip once the answer stopped mattering. Raises when it fires.
+
+ ``prefer_native`` tries /v1/chat/completions/input_tokens first (one round trip).
"""
if not self.is_loaded:
if strict:
@@ -38908,7 +38918,12 @@ class LlamaCppBackend:
tools = neutralize_tool_descriptions(tools, None, _profile)
try:
- with httpx.Client(timeout = 10, headers = self._auth_headers, trust_env = False) as client:
+ with httpx.Client(
+ timeout = 10,
+ headers = self._auth_headers,
+ trust_env = False,
+ verify = _count_ssl_context(),
+ ) as client:
def _tokenize(text: str) -> int:
r = client.post(
@@ -38955,6 +38970,22 @@ class LlamaCppBackend:
if continue_final_message:
template_body["continue_final_message"] = True
template_body["add_generation_prompt"] = False
+ if prefer_native:
+ if should_abort is not None and should_abort():
+ raise CountAborted()
+ try:
+ native = client.post(
+ f"{self.base_url}/v1/chat/completions/input_tokens",
+ json = template_body,
+ )
+ if native.status_code == 200:
+ count = native.json().get("input_tokens")
+ if type(count) is int and count > 0:
+ return count
+ except Exception:
+ pass
+ if should_abort is not None and should_abort():
+ raise CountAborted()
resp = client.post(
f"{self.base_url}/apply-template",
json = template_body,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 571b248b73a..5f7856f8a0e 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -2243,6 +2243,7 @@ def _openai_llama_admission_tokens(
image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,
injected_tools = None,
context_window: Optional[int] = None,
+ counted_prompt_tokens: Optional[int] = None,
vision: bool = True,
messages_override = None,
) -> Optional[int]:
@@ -2264,7 +2265,9 @@ def _openai_llama_admission_tokens(
messages = (
messages_override if messages_override is not None else getattr(payload, "messages", None)
)
- if isinstance(messages, list) and messages:
+ if counted_prompt_tokens is not None:
+ prompt_tokens = counted_prompt_tokens
+ elif isinstance(messages, list) and messages:
try:
estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(
messages, vision = vision
@@ -2372,6 +2375,7 @@ def _openai_llama_admission_reserve(
payload = None,
tool_loop: bool = False,
injected_tools = None,
+ counted_prompt_tokens: Optional[int] = None,
messages_override = None,
tokens = _TOKENS_UNSET,
) -> tuple[LlamaAdmissionReservation, LlamaAdmissionConfig]:
@@ -2379,7 +2383,16 @@ def _openai_llama_admission_reserve(
capacity = _openai_llama_admission_capacity(request, llama_backend)
key = str(getattr(llama_backend, "base_url", "llama-server"))
budget = _openai_llama_admission_budget(llama_backend)
- if tokens is _TOKENS_UNSET:
+ if counted_prompt_tokens is not None:
+ tokens = _openai_llama_admission_tokens(
+ payload,
+ counted_prompt_tokens = counted_prompt_tokens,
+ budget = budget,
+ capacity = capacity,
+ tool_loop = tool_loop,
+ context_window = _openai_llama_admission_context_window(llama_backend),
+ )
+ elif tokens is _TOKENS_UNSET:
tokens = (
_openai_llama_admission_estimate(
request = request,
@@ -2403,6 +2416,87 @@ def _openai_llama_admission_reserve(
return reservation, config
+def _count_gguf_admission_prompt(
+ llama_backend,
+ payload,
+ messages,
+ tools = None,
+ cancel_event = None,
+) -> int:
+ """Exact prompt count plus media allowance; the whole pool if counting fails
+ (the character estimate undercounts numeric text, #10671)."""
+ from core.inference.chat_template_helpers import trailing_assistant_text
+
+ budget = _openai_llama_admission_budget(llama_backend) or 0
+ try:
+ text_messages, images = _openai_llama_admission_messages_for_estimate(
+ messages, vision = bool(getattr(llama_backend, "is_vision", False))
+ )
+ count = llama_backend.count_chat_tokens(
+ text_messages,
+ tools = tools,
+ strict = True,
+ prefer_native = images == 0,
+ chat_template_kwargs = llama_backend._request_reasoning_kwargs(
+ payload.enable_thinking, payload.reasoning_effort, payload.preserve_thinking
+ ),
+ continue_final_message = _continue_final_message(payload)
+ and bool(trailing_assistant_text(messages)),
+ **({"should_abort": cancel_event.is_set} if cancel_event is not None else {}),
+ )
+ if type(count) is not int or count <= 0:
+ raise ValueError("Invalid prompt token count")
+ return count + _openai_llama_admission_media_tokens(
+ payload,
+ message_image_parts = images,
+ image_tokens = _openai_llama_admission_image_tokens(llama_backend),
+ message_video_clips = _conversation_video_clips(messages),
+ )
+ except Exception:
+ logger.debug("Prompt count unavailable; reserving the context pool", exc_info = True)
+ return budget
+
+
+async def _reserve_counted_gguf_chat(
+ *,
+ request,
+ llama_backend,
+ payload,
+ messages,
+ injected_tools = None,
+ tool_loop: bool = False,
+):
+ config = llama_admission_config_from_env()
+ if not (config.enabled and config.kv_budget):
+ return await _openai_llama_admission_reserve_async(
+ request = request,
+ llama_backend = llama_backend,
+ payload = payload,
+ tool_loop = tool_loop,
+ injected_tools = injected_tools,
+ )
+ identity = (
+ getattr(llama_backend, "base_url", None),
+ _openai_llama_admission_budget(llama_backend),
+ )
+ counted = await asyncio.to_thread(
+ _count_gguf_admission_prompt, llama_backend, payload, messages, injected_tools
+ )
+ if identity != (
+ getattr(llama_backend, "base_url", None),
+ _openai_llama_admission_budget(llama_backend),
+ ):
+ counted = _openai_llama_admission_budget(llama_backend)
+ return _openai_llama_admission_reserve(
+ request = request,
+ llama_backend = llama_backend,
+ payload = payload,
+ tool_loop = tool_loop,
+ injected_tools = injected_tools,
+ counted_prompt_tokens = counted,
+ )
+
+
async def _openai_llama_admission_reserve_async(
*,
request: Optional[Request],
@@ -2451,6 +2545,7 @@ def _openai_llama_admission_recost(
output_tokens: Optional[int] = None,
cancel_event = None,
injected_tools = None,
+ count_prepared_prompt: bool = False,
cache_is_empty: bool = False,
) -> None:
"""Charge a tool loop for what its conversation now is, not what it opened as.
@@ -2473,7 +2568,10 @@ def _openai_llama_admission_recost(
The gate exists because an idle slot's KV stays resident, which an erased slot's does
not, so yielding there hands back room that really is free.
"""
- if reservation is None:
+ if reservation is None or (cancel_event is not None and cancel_event.is_set()):
+ return
+ config = llama_admission_config_from_env()
+ if not (config.enabled and config.kv_budget):
return
try:
lease = reservation.lease_nowait()
@@ -2483,29 +2581,30 @@ def _openai_llama_admission_recost(
if not budget:
return
capacity = _openai_llama_admission_capacity(request, llama_backend)
- # Every term the OPENING reservation charges, charged again here. Counting fewer
- # things than the reservation it replaces would SHRINK a correctly sized lease --
- # and since the callback fires at the top of round zero, before any growth, it
- # would hand back room llama-server is already using.
- estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(
- conversation,
- vision = bool(getattr(llama_backend, "is_vision", False)),
- )
- prompt_tokens = estimate_messages_tokens_dense(estimate_messages)
- # Re-sent every round, so it belongs in every re-costing, not just the opening one.
- prompt_tokens += _openai_llama_admission_injected_tool_tokens(injected_tools)
- # Anthropic keeps `system` and `tools` out of the message list entirely, so for
- # that route this is most of the prompt.
- prompt_tokens += _openai_llama_admission_extra_prompt_tokens(payload)
- # mtmd embeddings, KV the message text cannot show: image parts compact to
- # "[image]" for the text estimate, so their real cost comes from the compaction
- # count. A screenshot tool adds more of them, so this grows with the rounds.
- prompt_tokens += _openai_llama_admission_media_tokens(
- payload,
- message_image_parts = message_image_parts,
- image_tokens = _openai_llama_admission_image_tokens(llama_backend),
- message_video_clips = _conversation_video_clips(conversation),
- )
+ if count_prepared_prompt:
+ prompt_tokens = _count_gguf_admission_prompt(
+ llama_backend, payload, conversation, injected_tools, cancel_event = cancel_event
+ )
+ if cancel_event is not None and cancel_event.is_set():
+ return
+ else:
+ # Charge every term the opening reservation did: fewer would shrink the lease at round
+ # zero and hand back room llama-server is already using.
+ estimate_messages, message_image_parts = _openai_llama_admission_messages_for_estimate(
+ conversation,
+ vision = bool(getattr(llama_backend, "is_vision", False)),
+ )
+ prompt_tokens = estimate_messages_tokens_dense(estimate_messages)
+ prompt_tokens += _openai_llama_admission_injected_tool_tokens(injected_tools)
+ # Anthropic keeps `system` and `tools` outside the message list.
+ prompt_tokens += _openai_llama_admission_extra_prompt_tokens(payload)
+ # mtmd embeddings: image parts compact to "[image]" in the text estimate.
+ prompt_tokens += _openai_llama_admission_media_tokens(
+ payload,
+ message_image_parts = message_image_parts,
+ image_tokens = _openai_llama_admission_image_tokens(llama_backend),
+ message_video_clips = _conversation_video_clips(conversation),
+ )
# Reading "Max" literally here would put the run back on the whole cache at its
# first round boundary.
share = max(1, budget // max(1, capacity))
@@ -26456,6 +26555,7 @@ async def produce_openai_chat_completions(
# loop down to its share on its very first round.
output_tokens = effective_max_tokens,
injected_tools = tools_to_use,
+ count_prepared_prompt = True,
# A round waiting for cache room must still answer Stop. Same event
# the loop polls each iteration, so a wait ends where a cancel would.
cancel_event = cancel_event,
@@ -26538,7 +26638,8 @@ async def produce_openai_chat_completions(
_tool_admission_mode = "chat_tool_stream" if payload.stream else "chat_tool_nonstream"
try:
- reservation, admission_config = await _openai_llama_admission_reserve_async(
+ reservation, admission_config = await _reserve_counted_gguf_chat(
+ messages = gguf_messages,
request = request,
llama_backend = llama_backend,
payload = payload,
@@ -27367,7 +27468,8 @@ async def produce_openai_chat_completions(
_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)
_tracker.__enter__()
try:
- reservation, admission_config = await _openai_llama_admission_reserve_async(
+ reservation, admission_config = await _reserve_counted_gguf_chat(
+ messages = gguf_messages,
request = request,
llama_backend = llama_backend,
payload = payload,
@@ -27384,6 +27486,9 @@ async def produce_openai_chat_completions(
)
api_monitor.fail(monitor_id, str(exc))
raise _openai_admission_http_exception(exc, status_code = 429)
+ except BaseException:
+ _tracker.__exit__(None, None, None)
+ raise
async def gguf_stream_chunks():
nonlocal _gguf_decode_finished
@@ -27711,7 +27816,8 @@ async def produce_openai_chat_completions(
)
else:
try:
- reservation, admission_config = await _openai_llama_admission_reserve_async(
+ reservation, admission_config = await _reserve_counted_gguf_chat(
+ messages = gguf_messages,
request = request,
llama_backend = llama_backend,
payload = payload,
diff --git a/studio/backend/tests/test_gguf_admission_token_count.py b/studio/backend/tests/test_gguf_admission_token_count.py
new file mode 100644
index 00000000000..afc2eeb4f8d
--- /dev/null
+++ b/studio/backend/tests/test_gguf_admission_token_count.py
@@ -0,0 +1,607 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Numeric text must not overcommit the shared KV pool through a character estimate."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+
+from core.inference.llama_admission import LlamaAdmissionQueue
+import routes.inference as inference
+
+
+def _llama_httpx():
+ # Patch the module llama_cpp resolves: another test can swap sys.modules["httpx"].
+ from core.inference import llama_cpp
+ return llama_cpp.httpx
+
+
+def _backend(count):
+ return SimpleNamespace(
+ base_url = "test-token-count",
+ context_length = 30000,
+ effective_parallel_slots = 3,
+ count_chat_tokens = Mock(return_value = count),
+ _request_reasoning_kwargs = Mock(return_value = {"enable_thinking": False}),
+ )
+
+
+def _payload():
+ return SimpleNamespace(
+ messages = [{"role": "user", "content": " ".join(map(str, range(1000, 5500)))}],
+ max_tokens = 1024,
+ enable_thinking = False,
+ reasoning_effort = None,
+ preserve_thinking = None,
+ )
+
+
+def test_numeric_prompts_wait_for_room_but_short_prompts_still_overlap(monkeypatch):
+ async def scenario(count):
+ queue = LlamaAdmissionQueue("counted")
+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)
+ backend = _backend(count)
+ payload = _payload()
+ reservations = []
+ for _ in range(3):
+ reservation, _ = await inference._reserve_counted_gguf_chat(
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ messages = payload.messages,
+ )
+ reservations.append(reservation)
+ leases = [r.lease_nowait() for r in reservations]
+ if count == 22560:
+ assert leases[0] is not None
+ assert leases[1:] == [None, None]
+ assert queue.snapshot().committed == 23584
+ leases[0].release()
+ await asyncio.sleep(0)
+ assert reservations[1].lease_nowait() is not None
+ else:
+ assert all(lease is not None for lease in leases)
+ assert backend.count_chat_tokens.call_count == 3
+
+ asyncio.run(scenario(22560))
+ asyncio.run(scenario(100))
+
+
+def test_counts_prepared_messages_and_tools_with_generation_options():
+ backend = _backend(22560)
+ payload = _payload()
+ prepared = [{"role": "system", "content": "Injected system instructions"}] + payload.messages
+ tools = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}]
+ count = inference._count_gguf_admission_prompt(backend, payload, prepared, tools)
+ assert count == 22560
+ args, kwargs = backend.count_chat_tokens.call_args
+ assert args[0] == prepared
+ assert kwargs == dict(
+ tools = tools,
+ strict = True,
+ prefer_native = True,
+ chat_template_kwargs = {"enable_thinking": False},
+ continue_final_message = False,
+ )
+ assert (
+ inference._openai_llama_admission_tokens(
+ payload,
+ budget = 30000,
+ capacity = 3,
+ injected_tools = tools,
+ counted_prompt_tokens = count,
+ )
+ == 23584
+ )
+
+
[email protected]("result", [0, -1, None, True, "22560"])
+def test_invalid_counts_reserve_the_entire_pool(result):
+ payload = _payload()
+ assert (
+ inference._count_gguf_admission_prompt(_backend(result), payload, payload.messages) == 30000
+ )
+
+
+def test_tokenizer_failure_reserves_the_entire_pool():
+ backend = _backend(0)
+ backend.count_chat_tokens.side_effect = RuntimeError("tokenizer unavailable")
+ payload = _payload()
+ assert inference._count_gguf_admission_prompt(backend, payload, payload.messages) == 30000
+
+
+def test_tool_recost_keeps_the_model_count(monkeypatch):
+ async def scenario():
+ queue = LlamaAdmissionQueue("recost")
+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)
+ backend = _backend(22560)
+ payload = _payload()
+ reservation, _ = await inference._reserve_counted_gguf_chat(
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ messages = payload.messages,
+ tool_loop = True,
+ )
+ assert reservation.lease_nowait() is not None
+ inference._openai_llama_admission_recost(
+ reservation,
+ payload.messages,
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ count_prepared_prompt = True,
+ )
+ assert queue.snapshot().committed == 23584
+
+ asyncio.run(scenario())
+
+
[email protected]("failed_path", ["legacy", "native", "unsupported", "invalid", "timeout"])
+def test_exact_count_endpoint_failover_preserves_short_chat_concurrency(monkeypatch, failed_path):
+ httpx = _llama_httpx()
+ from types import MethodType
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ calls = []
+
+ def respond(request):
+ path = request.url.path
+ calls.append(path)
+ if path == "/v1/chat/completions/input_tokens":
+ if failed_path == "timeout":
+ raise httpx.ReadTimeout("count timed out", request = request)
+ if failed_path in ("native", "unsupported"):
+ return httpx.Response(404 if failed_path == "unsupported" else 503)
+ if failed_path == "invalid":
+ return httpx.Response(200, json = {"input_tokens": True})
+ return httpx.Response(200, json = {"input_tokens": 20})
+ if failed_path == "legacy":
+ return httpx.Response(503)
+ if path == "/apply-template":
+ return httpx.Response(200, json = {"prompt": "rendered prompt"})
+ assert path == "/tokenize"
+ return httpx.Response(200, json = {"tokens": list(range(20))})
+
+ client_type = httpx.Client
+ transport = httpx.MockTransport(respond)
+ monkeypatch.setattr(httpx, "Client", lambda **kw: client_type(transport = transport, **kw))
+ backend = _backend(20)
+ backend.base_url = "http://llama.test"
+ backend.is_loaded = True
+ backend.markup_profile = None
+ backend._auth_headers = None
+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)
+ payload = _payload()
+ payload.messages = [{"role": "user", "content": "Hello"}]
+
+ async def scenario():
+ queue = LlamaAdmissionQueue("endpoint-failover")
+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)
+ reservations = await asyncio.gather(
+ *(
+ inference._reserve_counted_gguf_chat(
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ messages = payload.messages,
+ )
+ for _ in range(3)
+ )
+ )
+ leases = [reservation.lease_nowait() for reservation, _ in reservations]
+ try:
+ assert all(lease is not None for lease in leases)
+ assert queue.snapshot().committed == 3 * (20 + 1024)
+ finally:
+ for (reservation, _), lease in zip(reservations, leases):
+ if lease is not None:
+ lease.release()
+ else:
+ reservation.cancel()
+
+ asyncio.run(scenario())
+ assert calls.count("/v1/chat/completions/input_tokens") == 3
+ assert calls.count("/apply-template") == (0 if failed_path == "legacy" else 3)
+
+
+def test_native_count_preserves_rendering_options_and_tools(monkeypatch):
+ httpx = _llama_httpx()
+ import json
+ from types import MethodType
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ bodies = []
+
+ def respond(request):
+ assert request.url.path == "/v1/chat/completions/input_tokens"
+ bodies.append(json.loads(request.content))
+ return httpx.Response(200, json = {"input_tokens": 22560})
+
+ client_type = httpx.Client
+ monkeypatch.setattr(
+ httpx, "Client", lambda **kw: client_type(transport = httpx.MockTransport(respond), **kw)
+ )
+ backend = _backend(22560)
+ backend.base_url = "http://llama.test"
+ backend.is_loaded = True
+ backend.markup_profile = None
+ backend._auth_headers = None
+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)
+ messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "Partial"}]
+ tools = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}]
+ assert (
+ backend.count_chat_tokens(
+ messages,
+ system = "System instructions",
+ tools = tools,
+ strict = True,
+ chat_template_kwargs = {"enable_thinking": False},
+ continue_final_message = True,
+ prefer_native = True,
+ )
+ == 22560
+ )
+ assert bodies == [
+ {
+ "messages": [{"role": "system", "content": "System instructions"}] + messages,
+ "tools": tools,
+ "chat_template_kwargs": {"enable_thinking": False},
+ "continue_final_message": True,
+ "add_generation_prompt": False,
+ }
+ ]
+
+
+def test_all_count_endpoints_unavailable_still_reserves_pool(monkeypatch):
+ httpx = _llama_httpx()
+ from types import MethodType
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ client_type = httpx.Client
+ monkeypatch.setattr(
+ httpx,
+ "Client",
+ lambda **kw: client_type(
+ transport = httpx.MockTransport(lambda _: httpx.Response(503)), **kw
+ ),
+ )
+ backend = _backend(0)
+ backend.base_url = "http://llama.test"
+ backend.is_loaded = True
+ backend.markup_profile = None
+ backend._auth_headers = None
+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)
+ payload = _payload()
+ assert inference._count_gguf_admission_prompt(backend, payload, payload.messages) == 30000
+
+
+def test_media_keeps_existing_allowance_without_native_embedding_count():
+ backend = _backend(20)
+ payload = _payload()
+ payload.messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Describe this"},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}},
+ ],
+ }
+ ]
+ count = inference._count_gguf_admission_prompt(backend, payload, payload.messages)
+ assert count == 20 + inference._openai_llama_admission_image_tokens(backend)
+ assert backend.count_chat_tokens.call_args.kwargs["prefer_native"] is False
+
+
+def test_counting_recovers_after_a_complete_outage(monkeypatch):
+ async def scenario():
+ queue = LlamaAdmissionQueue("count-recovery")
+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)
+ backend = _backend(20)
+ payload = _payload()
+ payload.messages = [{"role": "user", "content": "Hello"}]
+ backend.count_chat_tokens.side_effect = RuntimeError("both count paths unavailable")
+ first, _ = await inference._reserve_counted_gguf_chat(
+ request = None, llama_backend = backend, payload = payload, messages = payload.messages
+ )
+ lease = first.lease_nowait()
+ assert lease is not None
+ assert queue.snapshot().committed == 30000
+ lease.release()
+ backend.count_chat_tokens.side_effect = None
+ reservations = [
+ (
+ await inference._reserve_counted_gguf_chat(
+ request = None, llama_backend = backend, payload = payload, messages = payload.messages
+ )
+ )[0]
+ for _ in range(3)
+ ]
+ leases = [reservation.lease_nowait() for reservation in reservations]
+ try:
+ assert all(lease is not None for lease in leases)
+ assert queue.snapshot().committed == 3 * (20 + 1024)
+ finally:
+ for reservation, lease in zip(reservations, leases):
+ if lease is not None:
+ lease.release()
+ else:
+ reservation.cancel()
+
+ asyncio.run(scenario())
+
+
+def test_exact_count_preserves_video_allowance(monkeypatch):
+ backend = _backend(20)
+ payload = _payload()
+ monkeypatch.setattr(inference, "_conversation_video_clips", lambda _: 2)
+ media = Mock(return_value = 4096)
+ monkeypatch.setattr(inference, "_openai_llama_admission_media_tokens", media)
+ assert inference._count_gguf_admission_prompt(backend, payload, payload.messages) == 4116
+ assert media.call_args.kwargs["message_video_clips"] == 2
+
+
+def test_prepared_message_parsing_runs_off_the_event_loop(monkeypatch):
+ import threading
+
+ loop_thread = threading.get_ident()
+ parse = inference._openai_llama_admission_messages_for_estimate
+ seen = []
+
+ def checked(*args, **kwargs):
+ seen.append(threading.get_ident())
+ assert threading.get_ident() != loop_thread
+ return parse(*args, **kwargs)
+
+ monkeypatch.setattr(inference, "_openai_llama_admission_messages_for_estimate", checked)
+
+ async def scenario():
+ payload = _payload()
+ reservation, _ = await inference._reserve_counted_gguf_chat(
+ request = None, llama_backend = _backend(20), payload = payload, messages = payload.messages
+ )
+ reservation.cancel()
+
+ asyncio.run(scenario())
+ assert seen and all(thread != loop_thread for thread in seen)
+
+
[email protected]("continue_request", [False, True])
+def test_tool_recost_rechecks_continuation_after_tool_result(monkeypatch, continue_request):
+ async def scenario():
+ queue = LlamaAdmissionQueue("continuation-recost")
+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)
+ backend = _backend(12000)
+ payload = _payload()
+ payload.continue_final_message = continue_request
+ payload.messages = [
+ {"role": "user", "content": "Look up the result"},
+ {"role": "assistant", "content": "Let me check."},
+ ]
+ reservation, _ = await inference._reserve_counted_gguf_chat(
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ messages = payload.messages,
+ tool_loop = True,
+ )
+ lease = reservation.lease_nowait()
+ assert lease is not None
+ try:
+ assert (
+ backend.count_chat_tokens.call_args.kwargs["continue_final_message"]
+ is continue_request
+ )
+ call = {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "lookup", "arguments": "{}"},
+ }
+ conversation = [
+ payload.messages[0],
+ {**payload.messages[1], "tool_calls": [call]},
+ {"role": "tool", "tool_call_id": "call_1", "content": "Found it"},
+ ]
+ inference._openai_llama_admission_recost(
+ reservation,
+ conversation,
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ output_tokens = 1024,
+ count_prepared_prompt = True,
+ )
+ assert backend.count_chat_tokens.call_count == 2
+ assert backend.count_chat_tokens.call_args.kwargs["continue_final_message"] is False
+ assert queue.snapshot().committed == 13024
+ finally:
+ lease.release()
+
+ asyncio.run(scenario())
+
+
+ "disabled", ["UNSLOTH_LLAMA_ADMISSION_CONTROL", "UNSLOTH_LLAMA_ADMISSION_KV_BUDGET"]
+)
+def test_disabled_admission_skips_tool_round_count(monkeypatch, disabled):
+ monkeypatch.setenv(disabled, "0")
+
+ async def scenario():
+ backend = _backend(20)
+ payload = _payload()
+ reservation, _ = await inference._reserve_counted_gguf_chat(
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ messages = payload.messages,
+ )
+ lease = reservation.lease_nowait()
+ assert lease is not None
+ try:
+ inference._openai_llama_admission_recost(
+ reservation,
+ payload.messages,
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ count_prepared_prompt = True,
+ )
+ backend.count_chat_tokens.assert_not_called()
+ finally:
+ lease.release()
+
+ asyncio.run(scenario())
+
+
[email protected]("cancel_during_count", [False, True])
+def test_cancelled_tool_round_does_not_count_or_recost(cancel_during_count):
+ import threading
+
+ cancel = threading.Event()
+ backend = _backend(20)
+ payload = _payload()
+ lease = Mock()
+ reservation = SimpleNamespace(lease_nowait = lambda: lease)
+ if cancel_during_count:
+
+ def count(*args, **kwargs):
+ assert kwargs["should_abort"]() is False
+ cancel.set()
+ assert kwargs["should_abort"]() is True
+ return 20
+
+ backend.count_chat_tokens.side_effect = count
+ else:
+ cancel.set()
+ inference._openai_llama_admission_recost(
+ reservation,
+ payload.messages,
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ cancel_event = cancel,
+ count_prepared_prompt = True,
+ )
+ assert backend.count_chat_tokens.call_count == int(cancel_during_count)
+ lease.recost_waiting.assert_not_called()
+
+
+def test_stop_after_native_count_failure_skips_fallback(monkeypatch):
+ import threading
+
+ httpx = _llama_httpx()
+ from types import MethodType
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ cancel = threading.Event()
+ calls = []
+
+ def respond(request):
+ calls.append(request.url.path)
+ cancel.set()
+ return httpx.Response(503)
+
+ client_type = httpx.Client
+ monkeypatch.setattr(
+ httpx, "Client", lambda **kw: client_type(transport = httpx.MockTransport(respond), **kw)
+ )
+ backend = _backend(20)
+ backend.base_url = "http://llama.test"
+ backend.is_loaded = True
+ backend.markup_profile = None
+ backend._auth_headers = None
+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)
+ payload = _payload()
+ lease = Mock()
+ inference._openai_llama_admission_recost(
+ SimpleNamespace(lease_nowait = lambda: lease),
+ payload.messages,
+ request = None,
+ llama_backend = backend,
+ payload = payload,
+ cancel_event = cancel,
+ count_prepared_prompt = True,
+ )
+ assert calls == ["/v1/chat/completions/input_tokens"]
+ lease.recost_waiting.assert_not_called()
+
+
+def test_cancel_during_stream_admission_exits_tracker(monkeypatch):
+ import threading
+ from starlette.requests import Request
+ from .llama_backend_double import FakeLlamaCppBackend
+ from models.inference import ChatCompletionRequest
+ from state import active_generations
+
+ entered = threading.Event()
+ release = threading.Event()
+
+ class Backend(FakeLlamaCppBackend):
+ context_length = 30000
+ base_url = "http://cancel-count-test"
+ effective_parallel_slots = 3
+
+ def _request_reasoning_kwargs(self, *args):
+ return {}
+
+ def count_chat_tokens(self, *args, **kwargs):
+ entered.set()
+ assert release.wait(5)
+ return 20
+
+ monkeypatch.setattr(inference, "get_llama_cpp_backend", lambda: Backend())
+ monkeypatch.setattr(inference, "_effective_enable_tools", lambda *a, **kw: False)
+ monkeypatch.setattr(inference, "_CANCEL_REGISTRY", {})
+ monkeypatch.setattr(active_generations, "_ACTIVE", {})
+ payload = ChatCompletionRequest(
+ messages = [{"role": "user", "content": "Hello"}],
+ stream = True,
+ enable_tools = False,
+ cancel_id = "count-cancel",
+ thread_id = "count-cancel",
+ )
+ request = Request(
+ {
+ "type": "http",
+ "method": "POST",
+ "path": "/chat/completions",
+ "headers": [],
+ "query_string": b"",
+ "state": {},
+ }
+ )
+
+ async def scenario():
+ task = asyncio.create_task(
+ inference.produce_openai_chat_completions(
+ payload,
+ request,
+ "test-user",
+ cancel_on_disconnect = True,
+ )
+ )
+ try:
+
+ async def wait_for_count():
+ while not entered.is_set():
+ if task.done():
+ await task
+ pytest.fail("Count was never entered")
+ await asyncio.sleep(0.001)
+
+ await asyncio.wait_for(wait_for_count(), timeout = 2)
+ assert active_generations.count() == 1
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert not inference._CANCEL_REGISTRY
+ assert active_generations.count() == 0
+ finally:
+ release.set()
+ task.cancel()
+ await asyncio.gather(task, return_exceptions = True)
+
+ asyncio.run(scenario())
diff --git a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
index 6901df1954d..05b38536f7b 100644
--- a/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
+++ b/studio/backend/tests/test_llama_admission_kv_budget_media_and_tools.py
@@ -575,12 +575,18 @@ class TestAnAudioTurnIsChargedByItsDuration:
def generate_chat_completion(self, **_kwargs):
yield "ok"
- charged, estimate = [], inference_route._openai_llama_admission_estimate
+ def count_chat_tokens(self, *_args, **_kwargs):
+ return 12
+
+ def _request_reasoning_kwargs(self, *_args):
+ return None
+
+ charged, tokens = [], inference_route._openai_llama_admission_tokens
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", _AudioGguf)
monkeypatch.setattr(
inference_route,
- "_openai_llama_admission_estimate",
- lambda **kw: charged.append(estimate(**kw)) or charged[-1],
+ "_openai_llama_admission_tokens",
+ lambda *a, **kw: charged.append(tokens(*a, **kw)) or charged[-1],
)
monkeypatch.setattr(
inference_route, "_decode_audio_mono", lambda _raw: (np.zeros(20 * 16000), 16000)
diff --git a/studio/backend/tests/test_mcp_images.py b/studio/backend/tests/test_mcp_images.py
index b341a283f61..625027012fc 100644
--- a/studio/backend/tests/test_mcp_images.py
+++ b/studio/backend/tests/test_mcp_images.py
@@ -2465,7 +2465,11 @@ def test_the_gguf_admission_estimate_parses_off_the_loop():
routes_src = inspect.getsource(inference_route)
assert "reservation, admission_config = _openai_llama_admission_reserve(" not in routes_src
- assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 7
+ assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 4
+ assert routes_src.count("await _reserve_counted_gguf_chat(") == 3
+ counted = inspect.getsource(inference_route._reserve_counted_gguf_chat)
+ assert "await asyncio.to_thread(" in counted
+ assert "_count_gguf_admission_prompt" in counted
wrapper = inspect.getsource(inference_route._openai_llama_admission_reserve_async)
assert "await asyncio.to_thread(\n _openai_llama_admission_estimate," in wrapper
assert "_messages_mention_mcp_images(" in wrapper