Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: reuse an identical llama-server prompt count within a tool round (#16756)
Daniel Han732c95b96bce
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 489bc54ed7a..865e8783ad2 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -10,6 +10,7 @@ OpenAI-compatible /v1/chat/completions endpoint.import astimport atexitimport bisect+import collectionsimport weakrefimport contextlibimport contextvars@@ -8160,6 +8161,62 @@ class CountAborted(Exception):_COUNT_TIMEOUT_FLOOR_S = 10.0_COUNT_TIMEOUT_CHARS_PER_S = 100_000+# Identical prompts recounted within a tool round; keyed per llama-server process.+_COUNT_MEMO_INIT_LOCK = threading.Lock()+_COUNT_MEMO_SIZE = 8+++def _count_memo_key(backend, *parts) -> Optional[bytes]:+ """Digest of everything the server render sees, scoped to the running llama-server process."""+ process = getattr(backend, "_process", None)+ try:+ payload = json.dumps(+ [+ getattr(backend, "base_url", None),+ getattr(backend, "_model_identifier", None),+ id(process),+ getattr(process, "pid", None),+ *parts,+ ],+ sort_keys = True,+ ensure_ascii = False,+ )+ except Exception: # unserialisable content: count it, never guess a key+ return None+ return hashlib.sha256(payload.encode("utf-8", "surrogatepass")).digest()+++def _count_memo(backend):+ memo = vars(backend).get("_count_memo_state")+ if memo is None:+ with _COUNT_MEMO_INIT_LOCK:+ memo = vars(backend).setdefault(+ "_count_memo_state", (threading.Lock(), collections.OrderedDict())+ )+ return memo+++def _count_memo_get(backend, key) -> Optional[int]:+ if key is None:+ return None+ lock, memo = _count_memo(backend)+ with lock:+ count = memo.get(key)+ if count is not None:+ memo.move_to_end(key)+ return count+++def _count_memo_put(backend, key, count) -> None:+ if key is None or type(count) is not int or count <= 0:+ return+ lock, memo = _count_memo(backend)+ with lock:+ memo[key] = count+ memo.move_to_end(key)+ while len(memo) > _COUNT_MEMO_SIZE:+ memo.popitem(last = False)+def _count_timeout(messages, system_text, tools) -> float:try:@@ -41522,6 +41579,35 @@ class LlamaCppBackend:tools = neutralize_tool_descriptions(tools, None, _profile)try:+ from core.inference.mcp_images import prepare_image_turn_boundaries++ template_messages = prepare_image_turn_boundaries(+ list(messages) if messages else [],+ getattr(self, "_chat_template_override", None)+ or getattr(self, "_chat_template", None),+ )+ if system_text:+ template_messages = [{"role": "system", "content": system_text}] + template_messages+ # Tool rounds recount the same prompt; admission (prefer_native) counts once and keeps its own failover.+ memo_key = (+ None+ if prefer_native+ else _count_memo_key(+ self,+ template_messages,+ tools,+ {+ key: template_options[key]+ for key in ("tool_choice", "parallel_tool_calls", "response_format")+ if template_options and key in template_options+ },+ chat_template_kwargs,+ continue_final_message,+ )+ )+ cached = _count_memo_get(self, memo_key)+ if cached is not None:+ return cachedwith httpx.Client(timeout = httpx.Timeout(_COUNT_TIMEOUT_FLOOR_S, read = _count_timeout(messages, system_text, tools)@@ -41548,18 +41634,6 @@ class LlamaCppBackend:return len(tokens)# 1. Try /apply-template to render the real chat prompt.- from core.inference.mcp_images import prepare_image_turn_boundaries-- template_messages = prepare_image_turn_boundaries(- list(messages) if messages else [],- getattr(self, "_chat_template_override", None)- or getattr(self, "_chat_template", None),- )- if system_text:- template_messages = [- {"role": "system", "content": system_text}- ] + template_messages-def _render_count(render_messages) -> tuple[Optional[int], bool]:"""(count or None, refused). Only a 500 is a template refusal; 503 = loading or no slot."""try:@@ -41634,6 +41708,7 @@ class LlamaCppBackend:if count is not None:self._empty_chat_render_refused = Trueif count is not None:+ _count_memo_put(self, memo_key, count)return count# The fallback drops role markers, special tokens and tool schemas (~30% of adiff --git a/studio/backend/tests/test_llama_count_memo.py b/studio/backend/tests/test_llama_count_memo.pynew file mode 100644index 00000000000..11a83d4d6ae--- /dev/null+++ b/studio/backend/tests/test_llama_count_memo.py@@ -0,0 +1,116 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A tool round near the window recounts the prompt it just fitted; one llama-server render serves both."""++import json+from types import MethodType, SimpleNamespace++import pytest+++def _setup(monkeypatch, *, status = 200):+ from core.inference import llama_cpp++ httpx = llama_cpp.httpx+ paths = []++ def respond(request):+ paths.append(request.url.path)+ if request.url.path == "/apply-template":+ if status != 200:+ return httpx.Response(status)+ body = json.loads(request.content)+ return httpx.Response(200, json = {"prompt": json.dumps(body)})+ if request.url.path == "/tokenize":+ text = json.loads(request.content)["content"]+ return httpx.Response(200, json = {"tokens": list(range(len(text) // 4))})+ return httpx.Response(404)++ client_type = httpx.Client+ monkeypatch.setattr(+ httpx, "Client", lambda **kw: client_type(transport = httpx.MockTransport(respond), **kw)+ )+ backend = SimpleNamespace(+ base_url = "http://llama.test",+ is_loaded = True,+ markup_profile = None,+ _auth_headers = None,+ _empty_chat_render_refused = False,+ _process = SimpleNamespace(pid = 4242),+ _model_identifier = "org/model-GGUF",+ )+ backend.count_chat_tokens = MethodType(llama_cpp.LlamaCppBackend.count_chat_tokens, backend)+ return backend, paths+++_MESSAGES = [{"role": "user", "content": "hello " * 200}]+_TOOLS = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}]+++def test_identical_recount_reuses_the_render(monkeypatch):+ backend, paths = _setup(monkeypatch)+ first = backend.count_chat_tokens(_MESSAGES, None, _TOOLS, strict = True)+ second = backend.count_chat_tokens(+ [dict(m) for m in _MESSAGES], None, list(_TOOLS), strict = True+ )+ assert first == second > 0+ assert paths == ["/apply-template", "/tokenize"]+++ "change",+ ["messages", "tools", "system", "kwargs", "continue", "options", "process", "model"],+)+def test_anything_the_render_sees_misses(monkeypatch, change):+ backend, paths = _setup(monkeypatch)+ backend.count_chat_tokens(_MESSAGES, None, _TOOLS, strict = True)+ messages, system, tools, kwargs = _MESSAGES, None, _TOOLS, {}+ if change == "messages":+ messages = _MESSAGES + [{"role": "assistant", "content": "x"}]+ elif change == "tools":+ tools = None+ elif change == "system":+ system = "Be brief."+ elif change == "kwargs":+ kwargs["chat_template_kwargs"] = {"enable_thinking": False}+ elif change == "continue":+ messages = _MESSAGES + [{"role": "assistant", "content": "Partial"}]+ backend.count_chat_tokens(messages, None, tools, strict = True)+ kwargs["continue_final_message"] = True+ elif change == "options":+ kwargs["template_options"] = {"tool_choice": "required"}+ elif change == "process":+ backend._process = SimpleNamespace(pid = 4343)+ elif change == "model":+ backend._model_identifier = "org/other-GGUF"+ before = paths.count("/tokenize")+ backend.count_chat_tokens(messages, system, tools, strict = True, **kwargs)+ assert paths.count("/tokenize") == before + 1+++def test_a_failed_render_is_not_remembered(monkeypatch):+ backend, paths = _setup(monkeypatch, status = 503)+ with pytest.raises(RuntimeError):+ backend.count_chat_tokens(_MESSAGES, None, _TOOLS, strict = True)+ # The non-strict text fallback is an approximation and must not be served later as a count.+ assert backend.count_chat_tokens(_MESSAGES, None, _TOOLS) > 0+ before = paths.count("/apply-template")+ backend.count_chat_tokens(_MESSAGES, None, _TOOLS)+ assert paths.count("/apply-template") == before + 1+++def test_admission_counts_are_not_memoised(monkeypatch):+ backend, paths = _setup(monkeypatch)+ for _ in range(2):+ backend.count_chat_tokens(_MESSAGES, None, _TOOLS, strict = True, prefer_native = True)+ assert paths.count("/v1/chat/completions/input_tokens") == 2+++def test_memo_stays_bounded(monkeypatch):+ from core.inference import llama_cpp++ backend, _ = _setup(monkeypatch)+ for i in range(3 * llama_cpp._COUNT_MEMO_SIZE):+ backend.count_chat_tokens([{"role": "user", "content": f"turn {i}"}], strict = True)+ assert len(llama_cpp._count_memo(backend)[1]) == llama_cpp._COUNT_MEMO_SIZE