Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Count prepared API prompts before GGUF admission (#15294)
Lee Jacksonb90dc406a2fe
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex afe92bcd4da..c9988583550 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -45642,6 +45642,7 @@ class LlamaCppBackend:continue_final_message: bool = False,should_abort = None,prefer_native: bool = False,+ template_options = None,) -> int:"""Count prompt tokens for a chat request via llama-server.@@ -45748,6 +45749,10 @@ class LlamaCppBackend:template_body = {"messages": render_messages}if tools:template_body["tools"] = llama_grammar_tools(tools)+ # These can affect the prompt rendered for a client-owned tool call.+ for key in ("tool_choice", "parallel_tool_calls", "response_format"):+ if template_options and key in template_options:+ template_body[key] = template_options[key]# Layered over the load-time --chat-template-kwargs: only keys sent here move.if chat_template_kwargs:template_body["chat_template_kwargs"] = chat_template_kwargsdiff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 1e2a3c2f269..e57cd5f08d3 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -2403,6 +2403,7 @@ def _openai_llama_admission_media_tokens(message_image_parts: int = 0,image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,message_video_clips: Optional[list[str]] = None,+ legacy_image_in_messages: bool = False,) -> int:"""Estimate media KV usage without charging transport bytes as prompt tokens.@@ -2417,7 +2418,7 @@ def _openai_llama_admission_media_tokens("""extra = 0extra += max(0, message_image_parts) * image_tokens- if _legacy_image_is_distinct(payload):+ if not legacy_image_in_messages and _legacy_image_is_distinct(payload):extra += image_tokensfor audio in _request_audio_clips(payload):if isinstance(audio, str) and audio:@@ -2674,6 +2675,8 @@ def _count_gguf_admission_prompt(messages,tools = None,cancel_event = None,+ *,+ prepared_body = None,) -> int:"""Exact prompt count plus media allowance; the whole pool if counting fails(the character estimate undercounts numeric text, #10671)."""@@ -2684,18 +2687,27 @@ def _count_gguf_admission_prompt(text_messages, images = _openai_llama_admission_messages_for_estimate(messages, vision = bool(getattr(llama_backend, "is_vision", False)))+ if prepared_body is None:+ template_kwargs = llama_backend._request_reasoning_kwargs(+ payload.enable_thinking,+ payload.reasoning_effort,+ _replay_preserve_thinking(llama_backend, payload, messages),+ )+ continue_final = (+ _continue_final_message(payload, thought = True)+ and trailing_assistant_resume_kind(messages) is not None+ )+ else:+ template_kwargs = prepared_body.get("chat_template_kwargs")+ continue_final = bool(prepared_body.get("continue_final_message"))count = llama_backend.count_chat_tokens(messages_without_unpriced_media(text_messages),tools = tools,strict = True,prefer_native = images == 0,- chat_template_kwargs = llama_backend._request_reasoning_kwargs(- payload.enable_thinking,- payload.reasoning_effort,- _replay_preserve_thinking(llama_backend, payload, messages),- ),- continue_final_message = _continue_final_message(payload, thought = True)- and trailing_assistant_resume_kind(messages) is not None,+ chat_template_kwargs = template_kwargs,+ continue_final_message = continue_final,+ **({"template_options": prepared_body} if prepared_body is not None else {}),**({"should_abort": cancel_event.is_set} if cancel_event is not None else {}),)if type(count) is not int or count <= 0:@@ -2705,6 +2717,7 @@ def _count_gguf_admission_prompt(message_image_parts = images,image_tokens = _openai_llama_admission_image_tokens(llama_backend),message_video_clips = _conversation_video_clips(messages),+ legacy_image_in_messages = prepared_body is not None,)except Exception:logger.debug("Prompt count unavailable; reserving the context pool", exc_info = True)@@ -2719,6 +2732,7 @@ async def _reserve_counted_gguf_chat(messages,injected_tools = None,tool_loop: bool = False,+ prepared_body = None,):config = llama_admission_config_from_env()if not (config.enabled and config.kv_budget):@@ -2734,7 +2748,12 @@ async def _reserve_counted_gguf_chat(_openai_llama_admission_budget(llama_backend),)counted = await asyncio.to_thread(- _count_gguf_admission_prompt, llama_backend, payload, messages, injected_tools+ _count_gguf_admission_prompt,+ llama_backend,+ payload,+ messages,+ injected_tools,+ **({"prepared_body": prepared_body} if prepared_body is not None else {}),)if identity != (getattr(llama_backend, "base_url", None),@@ -2751,6 +2770,30 @@ async def _reserve_counted_gguf_chat()+async def _reserve_counted_gguf_passthrough(+ *,+ request,+ llama_backend,+ payload,+ body = None,+):+ # Price the same prepared messages, tool schemas and template options that+ # generation sends. A character estimate misses numeric text and tool wrappers.+ if body is None:+ body = await _build_openai_passthrough_body_async(+ payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend+ )+ reservation, config = await _reserve_counted_gguf_chat(+ request = request,+ llama_backend = llama_backend,+ payload = payload,+ messages = body["messages"],+ injected_tools = body.get("tools"),+ prepared_body = body,+ )+ return reservation, config, body++async def _openai_llama_admission_reserve_async(*,request: Optional[Request],@@ -2800,6 +2843,7 @@ def _openai_llama_admission_recost(cancel_event = None,injected_tools = None,count_prepared_prompt: bool = False,+ prepared_body = None,cache_is_empty: bool = False,) -> None:"""Charge a tool loop for what its conversation now is, not what it opened as.@@ -2837,7 +2881,12 @@ def _openai_llama_admission_recost(capacity = _openai_llama_admission_capacity(request, llama_backend)if count_prepared_prompt:prompt_tokens = _count_gguf_admission_prompt(- llama_backend, payload, conversation, injected_tools, cancel_event = cancel_event+ llama_backend,+ payload,+ conversation,+ injected_tools,+ cancel_event = cancel_event,+ **({"prepared_body": prepared_body} if prepared_body is not None else {}),)if cancel_event is not None and cancel_event.is_set():return@@ -43781,13 +43830,14 @@ async def _responses_stream(cancel_event = threading.Event()_tracker = _TrackedCancel.for_payload(cancel_event, payload, resp_id)try:- reservation, admission_config = await _openai_llama_admission_reserve_async(+ reservation, admission_config, body = await _reserve_counted_gguf_passthrough(request = request,llama_backend = llama_backend,# chat_req, not payload: a ResponsesRequest carries `input` and# `max_output_tokens`, so the estimator found no `messages` and fell back to# one equal cache share no matter how large the request really was.payload = chat_req,+ body = body,)except LlamaAdmissionQueueFull as exc:_llama_admission_log(@@ -47488,33 +47538,56 @@ async def anthropic_messages(# estimate for the whole run._anthropic_admission_hold: dict = {"reservation": None}+ def _anthropic_admission_body(conversation, *, tool_loop = False):+ template_kwargs = _anthropic_count_template_kwargs(llama_backend, _reasoning_args)+ if client_tools or response_format is not None:+ # As the passthrough builds it: generation sends no parallel_tool_calls here.+ return _build_passthrough_payload(+ conversation,+ openai_tools,+ temperature,+ top_p,+ top_k,+ payload.max_tokens,+ payload.stream,+ tool_choice = openai_tool_choice,+ response_format = response_format,+ chat_template_kwargs = template_kwargs,+ backend_ctx = llama_backend.context_length,+ markup = getattr(llama_backend, "markup_profile", None),+ )+ return {+ "messages": conversation,+ "tools": openai_tools if tool_loop else None,+ "chat_template_kwargs": template_kwargs,+ }+def _anthropic_recost(conversation) -> None:+ body = _anthropic_admission_body(conversation, tool_loop = True)_openai_llama_admission_recost(_anthropic_admission_hold["reservation"],- conversation,+ body["messages"],request = request,llama_backend = llama_backend,- # `system` and `tools` live outside `messages` here, so without the payload a- # re-cost drops most of the prompt it is meant to charge.payload = payload,output_tokens = payload.max_tokens,- injected_tools = openai_tools,+ injected_tools = body.get("tools"),cancel_event = cancel_event,+ count_prepared_prompt = True,+ prepared_body = body,)async def _admitted_anthropic(coro, *, tool_loop: bool = False):try:- reservation, admission_config = await _openai_llama_admission_reserve_async(+ body = _anthropic_admission_body(openai_messages, tool_loop = tool_loop)+ reservation, admission_config = await _reserve_counted_gguf_chat(request = request,llama_backend = llama_backend,payload = payload,tool_loop = tool_loop,- # Only the tool branch resolves a catalogue; the plain branch sends none.- injected_tools = openai_tools if tool_loop else None,- # The list generation really runs on: already translated out of the- # Anthropic block shape and already promoted, so a replayed screenshot- # is charged as the image it becomes rather than as its base64.- messages_override = openai_messages,+ injected_tools = body.get("tools"),+ messages = body["messages"],+ prepared_body = body,)if tool_loop:_anthropic_admission_hold["reservation"] = reservation@@ -50144,7 +50217,7 @@ async def _openai_passthrough_stream(_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)_tracker.__enter__()try:- reservation, admission_config = await _openai_llama_admission_reserve_async(+ reservation, admission_config, prepared_body = await _reserve_counted_gguf_passthrough(request = request,llama_backend = llama_backend,payload = payload,@@ -50161,6 +50234,9 @@ async def _openai_passthrough_stream()api_monitor.fail(monitor_id, str(exc))raise _openai_admission_http_exception(exc, status_code = 429)+ except BaseException:+ _tracker.__exit__(None, None, None)+ raiselease = reservation.lease_nowait()if lease is not None:@@ -50193,6 +50269,7 @@ async def _openai_passthrough_stream(monitor_id = monitor_id,admission_lease = lease,tracker = _tracker,+ prepared_body = prepared_body,)admission_wait_started_at = time.monotonic()@@ -50244,6 +50321,7 @@ async def _openai_passthrough_stream(monitor_id = monitor_id,admission_lease = wait_item,tracker = _tracker,+ prepared_body = prepared_body,)admitted_started = Trueiterator = admitted_response.body_iterator@@ -50328,6 +50406,7 @@ async def _openai_passthrough_stream_admitted(*,admission_lease: LlamaAdmissionLease,tracker,+ prepared_body = None,):"""Streaming client-side pass-through after Unsloth granted an upstream slot.@@ -50378,9 +50457,11 @@ async def _openai_passthrough_stream_admitted(# Keep tracker cleanup paired if pre-header dispatch is cancelled.try:- body = await _build_openai_passthrough_body_async(- payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend- )+ body = prepared_body+ if body is None:+ body = await _build_openai_passthrough_body_async(+ payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend+ )# The body builder is allowlisted, so the caller's opt-in lives in the extra fields.client_wants_progress = bool((payload.model_extra or {}).get("return_progress"))body["return_progress"] = True@@ -51221,7 +51302,7 @@ async def _openai_passthrough_non_streaming():"""Non-streaming pass-through guarded by local llama-server admission."""try:- reservation, admission_config = await _openai_llama_admission_reserve_async(+ reservation, admission_config, prepared_body = await _reserve_counted_gguf_passthrough(request = request,llama_backend = llama_backend,payload = payload,@@ -51276,6 +51357,7 @@ async def _openai_passthrough_non_streaming(monitor_id = monitor_id,request = request,cancel_event = cancel_event,+ prepared_body = prepared_body,)except LlamaAdmissionTimeout as exc:_llama_admission_log(@@ -51322,6 +51404,7 @@ async def _openai_passthrough_non_streaming_upstream(*,request: Optional[Request] = None,cancel_event = None,+ prepared_body = None,):"""Non-streaming client-side pass-through for /v1/chat/completions.@@ -51331,9 +51414,11 @@ async def _openai_passthrough_non_streaming_upstream("""target_url = f"{llama_backend.base_url}/v1/chat/completions"upstream_headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend)- body = await _build_openai_passthrough_body_async(- payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend- )+ body = prepared_body+ if body is None:+ body = await _build_openai_passthrough_body_async(+ payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend+ )body["stream"] = Falsebody.pop("stream_options", None)diff --git a/studio/backend/tests/test_anthropic_admission.py b/studio/backend/tests/test_anthropic_admission.pyindex a44455799c0..898e7e678e1 100644--- a/studio/backend/tests/test_anthropic_admission.py+++ b/studio/backend/tests/test_anthropic_admission.py@@ -114,6 +114,94 @@ def _payload(**fields) -> AnthropicMessagesRequest:return AnthropicMessagesRequest(**base)+ "thinking", [{"type": "disabled"}, {"type": "enabled", "budget_tokens": 1024}]+)[email protected]("server_tools", [False, True])+def test_admission_counts_native_anthropic_thinking_mode(monkeypatch, thinking, server_tools):+ counts = []+ generated = []++ def count(*args, **kwargs):+ counts.append(kwargs)+ return 2++ backend = _install_backend(monkeypatch, count_tokens = count)++ def generate(**kwargs):+ generated.append(kwargs)+ yield "ok"++ def generate_tools(**kwargs):+ generated.append(kwargs)+ kwargs["on_conversation_grew"](kwargs["messages"])+ yield {"type": "content", "text": "ok"}++ backend.generate_chat_completion = generate+ backend.generate_chat_completion_with_tools = generate_tools+ backend._request_reasoning_kwargs = lambda enable, effort, preserve: {"enable_thinking": enable}+ payload = _payload(+ thinking = thinking,+ max_tokens = 1536,+ stream = False,+ enable_tools = server_tools,+ permission_mode = "off",+ )+ asyncio.run(inf_mod.anthropic_messages(payload, _Request(), "test-user"))+ expected = thinking["type"] == "enabled"+ assert generated[0]["enable_thinking"] is expected+ assert counts[0]["chat_template_kwargs"]["enable_thinking"] is expected+ assert all(item["chat_template_kwargs"]["enable_thinking"] is expected for item in counts)++[email protected]("mode", ["forced-tool", "schema"])+def test_admission_uses_normalized_anthropic_passthrough_body(monkeypatch, mode):+ backend = _install_backend(monkeypatch)+ backend.supports_tool_passthrough = True+ fields = {"stream": False}+ if mode == "forced-tool":+ fields.update(+ tools = [+ {+ "name": name,+ "description": name,+ "input_schema": {"type": "object", "properties": {}},+ }+ for name in ["first", "wanted"]+ ],+ tool_choice = {"type": "tool", "name": "wanted"},+ )+ else:+ fields["output_config"] = {+ "format": {+ "type": "json_schema",+ "schema": {+ "type": "object",+ "properties": {"ok": {"type": "boolean"}},+ "required": ["ok"],+ "additionalProperties": False,+ },+ }+ }+ captured = []++ async def reserve(**kwargs):+ captured.append(kwargs)+ raise RuntimeError("captured admission")++ monkeypatch.setattr(inf_mod, "_reserve_counted_gguf_chat", reserve)+ with pytest.raises(RuntimeError, match = "captured admission"):+ asyncio.run(inf_mod.anthropic_messages(_payload(**fields), _Request(), "test-user"))+ body = captured[0]["prepared_body"]+ if mode == "forced-tool":+ assert [tool["function"]["name"] for tool in body["tools"]] == ["wanted"]+ assert body["tool_choice"] == "required"+ else:+ assert body["response_format"]["type"] == "json_schema"+ assert captured[0]["messages"] == body["messages"]+ assert captured[0]["injected_tools"] == body.get("tools")++def _record_admission_logs(monkeypatch):"""Records via the logger, not caplog: this structlog logger never reaches the stdlib handlers."""records = []diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.pyindex f34065e7c60..6575bdd568f 100644--- a/studio/backend/tests/test_anthropic_messages.py+++ b/studio/backend/tests/test_anthropic_messages.py@@ -6181,7 +6181,7 @@ def test_the_anthropic_paths_promote_a_replayed_mcp_envelope():"the count endpoint prices the base64 the completion never sends, and must ""not do the Pillow work inline on the event loop")- assert "messages_override = openai_messages" in generate, (+ assert "_anthropic_admission_body(openai_messages" in generate, ("admission has to reserve against the translated list, or an Anthropic ""tool_result block prices its envelope as dense text")diff --git a/studio/backend/tests/test_gguf_admission_token_count.py b/studio/backend/tests/test_gguf_admission_token_count.pyindex 315cb30a2e1..4e3ef79f8fd 100644--- a/studio/backend/tests/test_gguf_admission_token_count.py+++ b/studio/backend/tests/test_gguf_admission_token_count.py@@ -39,6 +39,56 @@ def _payload():)+def test_prepared_legacy_image_is_charged_only_once():+ import base64+ import io+ from PIL import Image+ from models.inference import ChatCompletionRequest++ image = io.BytesIO()+ Image.new("RGB", (1, 1)).save(image, format = "PNG")+ backend = _backend(42)+ backend.is_vision = True+ payload = ChatCompletionRequest(+ messages = [{"role": "user", "content": "Describe this image"}],+ image_base64 = base64.b64encode(image.getvalue()).decode(),+ )+ messages = inference._openai_messages_for_passthrough(payload, vision = True)+ body = {"messages": messages}+ tokens = inference._count_gguf_admission_prompt(backend, payload, messages, prepared_body = body)+ assert tokens == 42 + inference._openai_llama_admission_image_tokens(backend)+++def test_passthrough_prices_the_prepared_prompt_before_upstream_dispatch(monkeypatch):+ async def scenario():+ backend = _backend(600)+ backend.context_length = 1024+ backend.effective_parallel_slots = 5+ payload = _payload()+ payload.max_tokens = 64+ body = {"messages": payload.messages, "max_tokens": 64, "chat_template_kwargs": {}}++ async def build(*args, **kwargs):+ return body++ async def upstream(*args, **kwargs):+ assert backend.count_chat_tokens.call_count == 1+ assert queue.snapshot().committed == 664+ assert kwargs["prepared_body"] is body+ return {"ok": True}++ queue = LlamaAdmissionQueue("passthrough-dispatch")+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)+ monkeypatch.setattr(inference, "_build_openai_passthrough_body_async", build)+ monkeypatch.setattr(inference, "_openai_passthrough_non_streaming_upstream", upstream)+ assert await inference._openai_passthrough_non_streaming(+ backend, payload, "test-model"+ ) == {"ok": True}+ assert queue.snapshot().active == 0++ asyncio.run(scenario())++def test_numeric_prompts_wait_for_room_but_short_prompts_still_overlap(monkeypatch):async def scenario(count):queue = LlamaAdmissionQueue("counted")@@ -113,6 +163,83 @@ def test_tokenizer_failure_reserves_the_entire_pool():assert inference._count_gguf_admission_prompt(backend, payload, payload.messages) == 30000+def test_passthrough_tool_prompt_queues_using_its_rendered_count(monkeypatch):+ async def scenario():+ queue = LlamaAdmissionQueue("passthrough-counted")+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)+ backend = _backend(600)+ backend.context_length = 1024+ backend.effective_parallel_slots = 5+ payload = _payload()+ payload.max_tokens = 64+ body = {+ "messages": [{"role": "user", "content": "Count numbers."}],+ "tools": [{"type": "function", "function": {"name": "unused", "parameters": {}}}],+ "tool_choice": "none",+ "chat_template_kwargs": {"enable_thinking": False},+ "max_tokens": 64,+ }+ reservations = []+ for _ in range(3):+ reservation, _, prepared = await inference._reserve_counted_gguf_passthrough(+ request = None, llama_backend = backend, payload = payload, body = body+ )+ assert prepared is body+ reservations.append(reservation)+ assert queue.snapshot().active == 1+ assert queue.snapshot().queued == 2+ assert queue.snapshot().committed == 664+ args, kwargs = backend.count_chat_tokens.call_args+ assert args[0] == body["messages"]+ assert kwargs["tools"] == body["tools"]+ assert kwargs["template_options"] is body+ assert kwargs["chat_template_kwargs"] == body["chat_template_kwargs"]+ reservations[0].lease_nowait().release()+ await asyncio.sleep(0)+ assert reservations[1].lease_nowait() is not None+ reservations[1].lease_nowait().release()+ reservations[2].cancel()+ assert queue.snapshot().active == 0+ assert queue.snapshot().queued == 0++ asyncio.run(scenario())+++def test_prompt_counter_forwards_passthrough_template_options(monkeypatch):+ from types import MethodType+ from core.inference.llama_cpp import LlamaCppBackend++ httpx = _llama_httpx()+ captured = []++ def respond(request):+ import json++ captured.append(json.loads(request.content))+ assert request.url.path == "/v1/chat/completions/input_tokens"+ return httpx.Response(200, json = {"input_tokens": 42})++ client_type = httpx.Client+ monkeypatch.setattr(+ httpx, "Client", lambda **kw: client_type(transport = httpx.MockTransport(respond), **kw)+ )+ backend = _backend(42)+ backend.base_url = "http://llama.test"+ backend.is_loaded = True+ backend.markup_profile = None+ backend._auth_headers = None+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)+ options = {"tool_choice": "none", "parallel_tool_calls": False, "messages": "ignored"}+ messages = [{"role": "user", "content": "Hello"}]+ assert (+ backend.count_chat_tokens(+ messages, strict = True, prefer_native = True, template_options = options+ )+ == 42+ )+ assert captured == [{"messages": messages, "tool_choice": "none", "parallel_tool_calls": False}]++def test_tool_recost_keeps_the_model_count(monkeypatch):async def scenario():queue = LlamaAdmissionQueue("recost")diff --git a/studio/backend/tests/test_mcp_images.py b/studio/backend/tests/test_mcp_images.pyindex e33127b24e1..7b6d70be18d 100644--- a/studio/backend/tests/test_mcp_images.py+++ b/studio/backend/tests/test_mcp_images.py@@ -2395,8 +2395,10 @@ def test_the_gguf_admission_estimate_parses_off_the_loop():routes_src = inspect.getsource(inference_route)assert "reservation, admission_config = _openai_llama_admission_reserve(" not in routes_src- assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 4- assert routes_src.count("await _reserve_counted_gguf_chat(") == 3+ assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 1+ assert routes_src.count("await _reserve_counted_gguf_chat(") == 5+ passthrough = inspect.getsource(inference_route._reserve_counted_gguf_passthrough)+ assert "await _reserve_counted_gguf_chat(" in passthroughcounted = inspect.getsource(inference_route._reserve_counted_gguf_chat)assert "await asyncio.to_thread(" in countedassert "_count_gguf_admission_prompt" in counted