Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Count prepared API prompts before GGUF admission (#15294)

Lee Jacksonb90dc406a2fe
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index afe92bcd4da..c9988583550 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -45642,6 +45642,7 @@ class LlamaCppBackend:
continue_final_message: bool = False,
should_abort = None,
prefer_native: bool = False,
+ template_options = None,
) -> int:
"""Count prompt tokens for a chat request via llama-server.
@@ -45748,6 +45749,10 @@ class LlamaCppBackend:
template_body = {"messages": render_messages}
if tools:
template_body["tools"] = llama_grammar_tools(tools)
+ # These can affect the prompt rendered for a client-owned tool call.
+ for key in ("tool_choice", "parallel_tool_calls", "response_format"):
+ if template_options and key in template_options:
+ template_body[key] = template_options[key]
# Layered over the load-time --chat-template-kwargs: only keys sent here move.
if chat_template_kwargs:
template_body["chat_template_kwargs"] = chat_template_kwargs
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 1e2a3c2f269..e57cd5f08d3 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -2403,6 +2403,7 @@ def _openai_llama_admission_media_tokens(
message_image_parts: int = 0,
image_tokens: int = _OPENAI_LLAMA_ADMISSION_IMAGE_TOKENS,
message_video_clips: Optional[list[str]] = None,
+ legacy_image_in_messages: bool = False,
) -> int:
"""Estimate media KV usage without charging transport bytes as prompt tokens.
@@ -2417,7 +2418,7 @@ def _openai_llama_admission_media_tokens(
"""
extra = 0
extra += max(0, message_image_parts) * image_tokens
- if _legacy_image_is_distinct(payload):
+ if not legacy_image_in_messages and _legacy_image_is_distinct(payload):
extra += image_tokens
for audio in _request_audio_clips(payload):
if isinstance(audio, str) and audio:
@@ -2674,6 +2675,8 @@ def _count_gguf_admission_prompt(
messages,
tools = None,
cancel_event = None,
+ *,
+ prepared_body = None,
) -> int:
"""Exact prompt count plus media allowance; the whole pool if counting fails
(the character estimate undercounts numeric text, #10671)."""
@@ -2684,18 +2687,27 @@ def _count_gguf_admission_prompt(
text_messages, images = _openai_llama_admission_messages_for_estimate(
messages, vision = bool(getattr(llama_backend, "is_vision", False))
)
+ if prepared_body is None:
+ template_kwargs = llama_backend._request_reasoning_kwargs(
+ payload.enable_thinking,
+ payload.reasoning_effort,
+ _replay_preserve_thinking(llama_backend, payload, messages),
+ )
+ continue_final = (
+ _continue_final_message(payload, thought = True)
+ and trailing_assistant_resume_kind(messages) is not None
+ )
+ else:
+ template_kwargs = prepared_body.get("chat_template_kwargs")
+ continue_final = bool(prepared_body.get("continue_final_message"))
count = llama_backend.count_chat_tokens(
messages_without_unpriced_media(text_messages),
tools = tools,
strict = True,
prefer_native = images == 0,
- chat_template_kwargs = llama_backend._request_reasoning_kwargs(
- payload.enable_thinking,
- payload.reasoning_effort,
- _replay_preserve_thinking(llama_backend, payload, messages),
- ),
- continue_final_message = _continue_final_message(payload, thought = True)
- and trailing_assistant_resume_kind(messages) is not None,
+ chat_template_kwargs = template_kwargs,
+ continue_final_message = continue_final,
+ **({"template_options": prepared_body} if prepared_body is not None else {}),
**({"should_abort": cancel_event.is_set} if cancel_event is not None else {}),
)
if type(count) is not int or count <= 0:
@@ -2705,6 +2717,7 @@ def _count_gguf_admission_prompt(
message_image_parts = images,
image_tokens = _openai_llama_admission_image_tokens(llama_backend),
message_video_clips = _conversation_video_clips(messages),
+ legacy_image_in_messages = prepared_body is not None,
)
except Exception:
logger.debug("Prompt count unavailable; reserving the context pool", exc_info = True)
@@ -2719,6 +2732,7 @@ async def _reserve_counted_gguf_chat(
messages,
injected_tools = None,
tool_loop: bool = False,
+ prepared_body = None,
):
config = llama_admission_config_from_env()
if not (config.enabled and config.kv_budget):
@@ -2734,7 +2748,12 @@ async def _reserve_counted_gguf_chat(
_openai_llama_admission_budget(llama_backend),
)
counted = await asyncio.to_thread(
- _count_gguf_admission_prompt, llama_backend, payload, messages, injected_tools
+ _count_gguf_admission_prompt,
+ llama_backend,
+ payload,
+ messages,
+ injected_tools,
+ **({"prepared_body": prepared_body} if prepared_body is not None else {}),
)
if identity != (
getattr(llama_backend, "base_url", None),
@@ -2751,6 +2770,30 @@ async def _reserve_counted_gguf_chat(
)
+async def _reserve_counted_gguf_passthrough(
+ *,
+ request,
+ llama_backend,
+ payload,
+ body = None,
+):
+ # Price the same prepared messages, tool schemas and template options that
+ # generation sends. A character estimate misses numeric text and tool wrappers.
+ if body is None:
+ body = await _build_openai_passthrough_body_async(
+ payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend
+ )
+ reservation, config = await _reserve_counted_gguf_chat(
+ request = request,
+ llama_backend = llama_backend,
+ payload = payload,
+ messages = body["messages"],
+ injected_tools = body.get("tools"),
+ prepared_body = body,
+ )
+ return reservation, config, body
+
+
async def _openai_llama_admission_reserve_async(
*,
request: Optional[Request],
@@ -2800,6 +2843,7 @@ def _openai_llama_admission_recost(
cancel_event = None,
injected_tools = None,
count_prepared_prompt: bool = False,
+ prepared_body = None,
cache_is_empty: bool = False,
) -> None:
"""Charge a tool loop for what its conversation now is, not what it opened as.
@@ -2837,7 +2881,12 @@ def _openai_llama_admission_recost(
capacity = _openai_llama_admission_capacity(request, llama_backend)
if count_prepared_prompt:
prompt_tokens = _count_gguf_admission_prompt(
- llama_backend, payload, conversation, injected_tools, cancel_event = cancel_event
+ llama_backend,
+ payload,
+ conversation,
+ injected_tools,
+ cancel_event = cancel_event,
+ **({"prepared_body": prepared_body} if prepared_body is not None else {}),
)
if cancel_event is not None and cancel_event.is_set():
return
@@ -43781,13 +43830,14 @@ async def _responses_stream(
cancel_event = threading.Event()
_tracker = _TrackedCancel.for_payload(cancel_event, payload, resp_id)
try:
- reservation, admission_config = await _openai_llama_admission_reserve_async(
+ reservation, admission_config, body = await _reserve_counted_gguf_passthrough(
request = request,
llama_backend = llama_backend,
# chat_req, not payload: a ResponsesRequest carries `input` and
# `max_output_tokens`, so the estimator found no `messages` and fell back to
# one equal cache share no matter how large the request really was.
payload = chat_req,
+ body = body,
)
except LlamaAdmissionQueueFull as exc:
_llama_admission_log(
@@ -47488,33 +47538,56 @@ async def anthropic_messages(
# estimate for the whole run.
_anthropic_admission_hold: dict = {"reservation": None}
+ def _anthropic_admission_body(conversation, *, tool_loop = False):
+ template_kwargs = _anthropic_count_template_kwargs(llama_backend, _reasoning_args)
+ if client_tools or response_format is not None:
+ # As the passthrough builds it: generation sends no parallel_tool_calls here.
+ return _build_passthrough_payload(
+ conversation,
+ openai_tools,
+ temperature,
+ top_p,
+ top_k,
+ payload.max_tokens,
+ payload.stream,
+ tool_choice = openai_tool_choice,
+ response_format = response_format,
+ chat_template_kwargs = template_kwargs,
+ backend_ctx = llama_backend.context_length,
+ markup = getattr(llama_backend, "markup_profile", None),
+ )
+ return {
+ "messages": conversation,
+ "tools": openai_tools if tool_loop else None,
+ "chat_template_kwargs": template_kwargs,
+ }
+
def _anthropic_recost(conversation) -> None:
+ body = _anthropic_admission_body(conversation, tool_loop = True)
_openai_llama_admission_recost(
_anthropic_admission_hold["reservation"],
- conversation,
+ body["messages"],
request = request,
llama_backend = llama_backend,
- # `system` and `tools` live outside `messages` here, so without the payload a
- # re-cost drops most of the prompt it is meant to charge.
payload = payload,
output_tokens = payload.max_tokens,
- injected_tools = openai_tools,
+ injected_tools = body.get("tools"),
cancel_event = cancel_event,
+ count_prepared_prompt = True,
+ prepared_body = body,
)
async def _admitted_anthropic(coro, *, tool_loop: bool = False):
try:
- reservation, admission_config = await _openai_llama_admission_reserve_async(
+ body = _anthropic_admission_body(openai_messages, tool_loop = tool_loop)
+ reservation, admission_config = await _reserve_counted_gguf_chat(
request = request,
llama_backend = llama_backend,
payload = payload,
tool_loop = tool_loop,
- # Only the tool branch resolves a catalogue; the plain branch sends none.
- injected_tools = openai_tools if tool_loop else None,
- # The list generation really runs on: already translated out of the
- # Anthropic block shape and already promoted, so a replayed screenshot
- # is charged as the image it becomes rather than as its base64.
- messages_override = openai_messages,
+ injected_tools = body.get("tools"),
+ messages = body["messages"],
+ prepared_body = body,
)
if tool_loop:
_anthropic_admission_hold["reservation"] = reservation
@@ -50144,7 +50217,7 @@ async def _openai_passthrough_stream(
_tracker = _TrackedCancel.for_payload(cancel_event, payload, *_cancel_keys)
_tracker.__enter__()
try:
- reservation, admission_config = await _openai_llama_admission_reserve_async(
+ reservation, admission_config, prepared_body = await _reserve_counted_gguf_passthrough(
request = request,
llama_backend = llama_backend,
payload = payload,
@@ -50161,6 +50234,9 @@ async def _openai_passthrough_stream(
)
api_monitor.fail(monitor_id, str(exc))
raise _openai_admission_http_exception(exc, status_code = 429)
+ except BaseException:
+ _tracker.__exit__(None, None, None)
+ raise
lease = reservation.lease_nowait()
if lease is not None:
@@ -50193,6 +50269,7 @@ async def _openai_passthrough_stream(
monitor_id = monitor_id,
admission_lease = lease,
tracker = _tracker,
+ prepared_body = prepared_body,
)
admission_wait_started_at = time.monotonic()
@@ -50244,6 +50321,7 @@ async def _openai_passthrough_stream(
monitor_id = monitor_id,
admission_lease = wait_item,
tracker = _tracker,
+ prepared_body = prepared_body,
)
admitted_started = True
iterator = admitted_response.body_iterator
@@ -50328,6 +50406,7 @@ async def _openai_passthrough_stream_admitted(
*,
admission_lease: LlamaAdmissionLease,
tracker,
+ prepared_body = None,
):
"""Streaming client-side pass-through after Unsloth granted an upstream slot.
@@ -50378,9 +50457,11 @@ async def _openai_passthrough_stream_admitted(
# Keep tracker cleanup paired if pre-header dispatch is cancelled.
try:
- body = await _build_openai_passthrough_body_async(
- payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend
- )
+ body = prepared_body
+ if body is None:
+ body = await _build_openai_passthrough_body_async(
+ payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend
+ )
# The body builder is allowlisted, so the caller's opt-in lives in the extra fields.
client_wants_progress = bool((payload.model_extra or {}).get("return_progress"))
body["return_progress"] = True
@@ -51221,7 +51302,7 @@ async def _openai_passthrough_non_streaming(
):
"""Non-streaming pass-through guarded by local llama-server admission."""
try:
- reservation, admission_config = await _openai_llama_admission_reserve_async(
+ reservation, admission_config, prepared_body = await _reserve_counted_gguf_passthrough(
request = request,
llama_backend = llama_backend,
payload = payload,
@@ -51276,6 +51357,7 @@ async def _openai_passthrough_non_streaming(
monitor_id = monitor_id,
request = request,
cancel_event = cancel_event,
+ prepared_body = prepared_body,
)
except LlamaAdmissionTimeout as exc:
_llama_admission_log(
@@ -51322,6 +51404,7 @@ async def _openai_passthrough_non_streaming_upstream(
*,
request: Optional[Request] = None,
cancel_event = None,
+ prepared_body = None,
):
"""Non-streaming client-side pass-through for /v1/chat/completions.
@@ -51331,9 +51414,11 @@ async def _openai_passthrough_non_streaming_upstream(
"""
target_url = f"{llama_backend.base_url}/v1/chat/completions"
upstream_headers = _openai_passthrough_upstream_headers(llama_backend = llama_backend)
- body = await _build_openai_passthrough_body_async(
- payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend
- )
+ body = prepared_body
+ if body is None:
+ body = await _build_openai_passthrough_body_async(
+ payload, backend_ctx = llama_backend.context_length, llama_backend = llama_backend
+ )
body["stream"] = False
body.pop("stream_options", None)
diff --git a/studio/backend/tests/test_anthropic_admission.py b/studio/backend/tests/test_anthropic_admission.py
index a44455799c0..898e7e678e1 100644
--- a/studio/backend/tests/test_anthropic_admission.py
+++ b/studio/backend/tests/test_anthropic_admission.py
@@ -114,6 +114,94 @@ def _payload(**fields) -> AnthropicMessagesRequest:
return AnthropicMessagesRequest(**base)
+ "thinking", [{"type": "disabled"}, {"type": "enabled", "budget_tokens": 1024}]
+)
[email protected]("server_tools", [False, True])
+def test_admission_counts_native_anthropic_thinking_mode(monkeypatch, thinking, server_tools):
+ counts = []
+ generated = []
+
+ def count(*args, **kwargs):
+ counts.append(kwargs)
+ return 2
+
+ backend = _install_backend(monkeypatch, count_tokens = count)
+
+ def generate(**kwargs):
+ generated.append(kwargs)
+ yield "ok"
+
+ def generate_tools(**kwargs):
+ generated.append(kwargs)
+ kwargs["on_conversation_grew"](kwargs["messages"])
+ yield {"type": "content", "text": "ok"}
+
+ backend.generate_chat_completion = generate
+ backend.generate_chat_completion_with_tools = generate_tools
+ backend._request_reasoning_kwargs = lambda enable, effort, preserve: {"enable_thinking": enable}
+ payload = _payload(
+ thinking = thinking,
+ max_tokens = 1536,
+ stream = False,
+ enable_tools = server_tools,
+ permission_mode = "off",
+ )
+ asyncio.run(inf_mod.anthropic_messages(payload, _Request(), "test-user"))
+ expected = thinking["type"] == "enabled"
+ assert generated[0]["enable_thinking"] is expected
+ assert counts[0]["chat_template_kwargs"]["enable_thinking"] is expected
+ assert all(item["chat_template_kwargs"]["enable_thinking"] is expected for item in counts)
+
+
[email protected]("mode", ["forced-tool", "schema"])
+def test_admission_uses_normalized_anthropic_passthrough_body(monkeypatch, mode):
+ backend = _install_backend(monkeypatch)
+ backend.supports_tool_passthrough = True
+ fields = {"stream": False}
+ if mode == "forced-tool":
+ fields.update(
+ tools = [
+ {
+ "name": name,
+ "description": name,
+ "input_schema": {"type": "object", "properties": {}},
+ }
+ for name in ["first", "wanted"]
+ ],
+ tool_choice = {"type": "tool", "name": "wanted"},
+ )
+ else:
+ fields["output_config"] = {
+ "format": {
+ "type": "json_schema",
+ "schema": {
+ "type": "object",
+ "properties": {"ok": {"type": "boolean"}},
+ "required": ["ok"],
+ "additionalProperties": False,
+ },
+ }
+ }
+ captured = []
+
+ async def reserve(**kwargs):
+ captured.append(kwargs)
+ raise RuntimeError("captured admission")
+
+ monkeypatch.setattr(inf_mod, "_reserve_counted_gguf_chat", reserve)
+ with pytest.raises(RuntimeError, match = "captured admission"):
+ asyncio.run(inf_mod.anthropic_messages(_payload(**fields), _Request(), "test-user"))
+ body = captured[0]["prepared_body"]
+ if mode == "forced-tool":
+ assert [tool["function"]["name"] for tool in body["tools"]] == ["wanted"]
+ assert body["tool_choice"] == "required"
+ else:
+ assert body["response_format"]["type"] == "json_schema"
+ assert captured[0]["messages"] == body["messages"]
+ assert captured[0]["injected_tools"] == body.get("tools")
+
+
def _record_admission_logs(monkeypatch):
"""Records via the logger, not caplog: this structlog logger never reaches the stdlib handlers."""
records = []
diff --git a/studio/backend/tests/test_anthropic_messages.py b/studio/backend/tests/test_anthropic_messages.py
index f34065e7c60..6575bdd568f 100644
--- a/studio/backend/tests/test_anthropic_messages.py
+++ b/studio/backend/tests/test_anthropic_messages.py
@@ -6181,7 +6181,7 @@ def test_the_anthropic_paths_promote_a_replayed_mcp_envelope():
"the count endpoint prices the base64 the completion never sends, and must "
"not do the Pillow work inline on the event loop"
)
- assert "messages_override = openai_messages" in generate, (
+ assert "_anthropic_admission_body(openai_messages" in generate, (
"admission has to reserve against the translated list, or an Anthropic "
"tool_result block prices its envelope as dense text"
)
diff --git a/studio/backend/tests/test_gguf_admission_token_count.py b/studio/backend/tests/test_gguf_admission_token_count.py
index 315cb30a2e1..4e3ef79f8fd 100644
--- a/studio/backend/tests/test_gguf_admission_token_count.py
+++ b/studio/backend/tests/test_gguf_admission_token_count.py
@@ -39,6 +39,56 @@ def _payload():
)
+def test_prepared_legacy_image_is_charged_only_once():
+ import base64
+ import io
+ from PIL import Image
+ from models.inference import ChatCompletionRequest
+
+ image = io.BytesIO()
+ Image.new("RGB", (1, 1)).save(image, format = "PNG")
+ backend = _backend(42)
+ backend.is_vision = True
+ payload = ChatCompletionRequest(
+ messages = [{"role": "user", "content": "Describe this image"}],
+ image_base64 = base64.b64encode(image.getvalue()).decode(),
+ )
+ messages = inference._openai_messages_for_passthrough(payload, vision = True)
+ body = {"messages": messages}
+ tokens = inference._count_gguf_admission_prompt(backend, payload, messages, prepared_body = body)
+ assert tokens == 42 + inference._openai_llama_admission_image_tokens(backend)
+
+
+def test_passthrough_prices_the_prepared_prompt_before_upstream_dispatch(monkeypatch):
+ async def scenario():
+ backend = _backend(600)
+ backend.context_length = 1024
+ backend.effective_parallel_slots = 5
+ payload = _payload()
+ payload.max_tokens = 64
+ body = {"messages": payload.messages, "max_tokens": 64, "chat_template_kwargs": {}}
+
+ async def build(*args, **kwargs):
+ return body
+
+ async def upstream(*args, **kwargs):
+ assert backend.count_chat_tokens.call_count == 1
+ assert queue.snapshot().committed == 664
+ assert kwargs["prepared_body"] is body
+ return {"ok": True}
+
+ queue = LlamaAdmissionQueue("passthrough-dispatch")
+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)
+ monkeypatch.setattr(inference, "_build_openai_passthrough_body_async", build)
+ monkeypatch.setattr(inference, "_openai_passthrough_non_streaming_upstream", upstream)
+ assert await inference._openai_passthrough_non_streaming(
+ backend, payload, "test-model"
+ ) == {"ok": True}
+ assert queue.snapshot().active == 0
+
+ asyncio.run(scenario())
+
+
def test_numeric_prompts_wait_for_room_but_short_prompts_still_overlap(monkeypatch):
async def scenario(count):
queue = LlamaAdmissionQueue("counted")
@@ -113,6 +163,83 @@ def test_tokenizer_failure_reserves_the_entire_pool():
assert inference._count_gguf_admission_prompt(backend, payload, payload.messages) == 30000
+def test_passthrough_tool_prompt_queues_using_its_rendered_count(monkeypatch):
+ async def scenario():
+ queue = LlamaAdmissionQueue("passthrough-counted")
+ monkeypatch.setattr(inference, "get_llama_admission_queue", lambda _: queue)
+ backend = _backend(600)
+ backend.context_length = 1024
+ backend.effective_parallel_slots = 5
+ payload = _payload()
+ payload.max_tokens = 64
+ body = {
+ "messages": [{"role": "user", "content": "Count numbers."}],
+ "tools": [{"type": "function", "function": {"name": "unused", "parameters": {}}}],
+ "tool_choice": "none",
+ "chat_template_kwargs": {"enable_thinking": False},
+ "max_tokens": 64,
+ }
+ reservations = []
+ for _ in range(3):
+ reservation, _, prepared = await inference._reserve_counted_gguf_passthrough(
+ request = None, llama_backend = backend, payload = payload, body = body
+ )
+ assert prepared is body
+ reservations.append(reservation)
+ assert queue.snapshot().active == 1
+ assert queue.snapshot().queued == 2
+ assert queue.snapshot().committed == 664
+ args, kwargs = backend.count_chat_tokens.call_args
+ assert args[0] == body["messages"]
+ assert kwargs["tools"] == body["tools"]
+ assert kwargs["template_options"] is body
+ assert kwargs["chat_template_kwargs"] == body["chat_template_kwargs"]
+ reservations[0].lease_nowait().release()
+ await asyncio.sleep(0)
+ assert reservations[1].lease_nowait() is not None
+ reservations[1].lease_nowait().release()
+ reservations[2].cancel()
+ assert queue.snapshot().active == 0
+ assert queue.snapshot().queued == 0
+
+ asyncio.run(scenario())
+
+
+def test_prompt_counter_forwards_passthrough_template_options(monkeypatch):
+ from types import MethodType
+ from core.inference.llama_cpp import LlamaCppBackend
+
+ httpx = _llama_httpx()
+ captured = []
+
+ def respond(request):
+ import json
+
+ captured.append(json.loads(request.content))
+ assert request.url.path == "/v1/chat/completions/input_tokens"
+ return httpx.Response(200, json = {"input_tokens": 42})
+
+ client_type = httpx.Client
+ monkeypatch.setattr(
+ httpx, "Client", lambda **kw: client_type(transport = httpx.MockTransport(respond), **kw)
+ )
+ backend = _backend(42)
+ backend.base_url = "http://llama.test"
+ backend.is_loaded = True
+ backend.markup_profile = None
+ backend._auth_headers = None
+ backend.count_chat_tokens = MethodType(LlamaCppBackend.count_chat_tokens, backend)
+ options = {"tool_choice": "none", "parallel_tool_calls": False, "messages": "ignored"}
+ messages = [{"role": "user", "content": "Hello"}]
+ assert (
+ backend.count_chat_tokens(
+ messages, strict = True, prefer_native = True, template_options = options
+ )
+ == 42
+ )
+ assert captured == [{"messages": messages, "tool_choice": "none", "parallel_tool_calls": False}]
+
+
def test_tool_recost_keeps_the_model_count(monkeypatch):
async def scenario():
queue = LlamaAdmissionQueue("recost")
diff --git a/studio/backend/tests/test_mcp_images.py b/studio/backend/tests/test_mcp_images.py
index e33127b24e1..7b6d70be18d 100644
--- a/studio/backend/tests/test_mcp_images.py
+++ b/studio/backend/tests/test_mcp_images.py
@@ -2395,8 +2395,10 @@ def test_the_gguf_admission_estimate_parses_off_the_loop():
routes_src = inspect.getsource(inference_route)
assert "reservation, admission_config = _openai_llama_admission_reserve(" not in routes_src
- assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 4
- assert routes_src.count("await _reserve_counted_gguf_chat(") == 3
+ assert routes_src.count("await _openai_llama_admission_reserve_async(") >= 1
+ assert routes_src.count("await _reserve_counted_gguf_chat(") == 5
+ passthrough = inspect.getsource(inference_route._reserve_counted_gguf_passthrough)
+ assert "await _reserve_counted_gguf_chat(" in passthrough
counted = inspect.getsource(inference_route._reserve_counted_gguf_chat)
assert "await asyncio.to_thread(" in counted
assert "_count_gguf_admission_prompt" in counted