Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop chats with images from blocking other chats on GGUF models (#12236)

Nilay6aac263baa86
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index e520a0bf13e..0cc3661f50d 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -105,6 +105,7 @@ from core.inference.context_window import (
estimate_message_tokens as _estimate_message_tokens,
estimate_messages_tokens as _estimate_messages_tokens,
estimate_messages_tokens_dense,
+ messages_without_unpriced_media,
truncate_oldest_messages as _truncate_oldest_messages,
)
from core.inference.llama_tool_schema import llama_grammar_tools
@@ -2472,7 +2473,7 @@ def _count_gguf_admission_prompt(
messages, vision = bool(getattr(llama_backend, "is_vision", False))
)
count = llama_backend.count_chat_tokens(
- text_messages,
+ messages_without_unpriced_media(text_messages),
tools = tools,
strict = True,
prefer_native = images == 0,
diff --git a/studio/backend/tests/test_gguf_admission_token_count.py b/studio/backend/tests/test_gguf_admission_token_count.py
index afc2eeb4f8d..b0f0e2718e9 100644
--- a/studio/backend/tests/test_gguf_admission_token_count.py
+++ b/studio/backend/tests/test_gguf_admission_token_count.py
@@ -279,7 +279,7 @@ def test_all_count_endpoints_unavailable_still_reserves_pool(monkeypatch):
assert inference._count_gguf_admission_prompt(backend, payload, payload.messages) == 30000
-def test_media_keeps_existing_allowance_without_native_embedding_count():
+def test_media_is_charged_by_allowance_and_not_sent_to_the_count():
backend = _backend(20)
payload = _payload()
payload.messages = [
@@ -294,6 +294,9 @@ def test_media_keeps_existing_allowance_without_native_embedding_count():
count = inference._count_gguf_admission_prompt(backend, payload, payload.messages)
assert count == 20 + inference._openai_llama_admission_image_tokens(backend)
assert backend.count_chat_tokens.call_args.kwargs["prefer_native"] is False
+ # llama-server rejects the "[image]" placeholder, and a failed count reserves the whole pool.
+ sent = backend.count_chat_tokens.call_args.args[0]
+ assert [part["type"] for part in sent[0]["content"]] == ["text"]
def test_counting_recovers_after_a_complete_outage(monkeypatch):