Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: reuse an identical llama-server prompt count within a tool round (#16756)

Daniel Han732c95b96bce
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 489bc54ed7a..865e8783ad2 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -10,6 +10,7 @@ OpenAI-compatible /v1/chat/completions endpoint.
import ast
import atexit
import bisect
+import collections
import weakref
import contextlib
import contextvars
@@ -8160,6 +8161,62 @@ class CountAborted(Exception):
_COUNT_TIMEOUT_FLOOR_S = 10.0
_COUNT_TIMEOUT_CHARS_PER_S = 100_000
+# Identical prompts recounted within a tool round; keyed per llama-server process.
+_COUNT_MEMO_INIT_LOCK = threading.Lock()
+_COUNT_MEMO_SIZE = 8
+
+
+def _count_memo_key(backend, *parts) -> Optional[bytes]:
+ """Digest of everything the server render sees, scoped to the running llama-server process."""
+ process = getattr(backend, "_process", None)
+ try:
+ payload = json.dumps(
+ [
+ getattr(backend, "base_url", None),
+ getattr(backend, "_model_identifier", None),
+ id(process),
+ getattr(process, "pid", None),
+ *parts,
+ ],
+ sort_keys = True,
+ ensure_ascii = False,
+ )
+ except Exception: # unserialisable content: count it, never guess a key
+ return None
+ return hashlib.sha256(payload.encode("utf-8", "surrogatepass")).digest()
+
+
+def _count_memo(backend):
+ memo = vars(backend).get("_count_memo_state")
+ if memo is None:
+ with _COUNT_MEMO_INIT_LOCK:
+ memo = vars(backend).setdefault(
+ "_count_memo_state", (threading.Lock(), collections.OrderedDict())
+ )
+ return memo
+
+
+def _count_memo_get(backend, key) -> Optional[int]:
+ if key is None:
+ return None
+ lock, memo = _count_memo(backend)
+ with lock:
+ count = memo.get(key)
+ if count is not None:
+ memo.move_to_end(key)
+ return count
+
+
+def _count_memo_put(backend, key, count) -> None:
+ if key is None or type(count) is not int or count <= 0:
+ return
+ lock, memo = _count_memo(backend)
+ with lock:
+ memo[key] = count
+ memo.move_to_end(key)
+ while len(memo) > _COUNT_MEMO_SIZE:
+ memo.popitem(last = False)
+
def _count_timeout(messages, system_text, tools) -> float:
try:
@@ -41522,6 +41579,35 @@ class LlamaCppBackend:
tools = neutralize_tool_descriptions(tools, None, _profile)
try:
+ from core.inference.mcp_images import prepare_image_turn_boundaries
+
+ template_messages = prepare_image_turn_boundaries(
+ list(messages) if messages else [],
+ getattr(self, "_chat_template_override", None)
+ or getattr(self, "_chat_template", None),
+ )
+ if system_text:
+ template_messages = [{"role": "system", "content": system_text}] + template_messages
+ # Tool rounds recount the same prompt; admission (prefer_native) counts once and keeps its own failover.
+ memo_key = (
+ None
+ if prefer_native
+ else _count_memo_key(
+ self,
+ template_messages,
+ tools,
+ {
+ key: template_options[key]
+ for key in ("tool_choice", "parallel_tool_calls", "response_format")
+ if template_options and key in template_options
+ },
+ chat_template_kwargs,
+ continue_final_message,
+ )
+ )
+ cached = _count_memo_get(self, memo_key)
+ if cached is not None:
+ return cached
with httpx.Client(
timeout = httpx.Timeout(
_COUNT_TIMEOUT_FLOOR_S, read = _count_timeout(messages, system_text, tools)
@@ -41548,18 +41634,6 @@ class LlamaCppBackend:
return len(tokens)
# 1. Try /apply-template to render the real chat prompt.
- from core.inference.mcp_images import prepare_image_turn_boundaries
-
- template_messages = prepare_image_turn_boundaries(
- list(messages) if messages else [],
- getattr(self, "_chat_template_override", None)
- or getattr(self, "_chat_template", None),
- )
- if system_text:
- template_messages = [
- {"role": "system", "content": system_text}
- ] + template_messages
-
def _render_count(render_messages) -> tuple[Optional[int], bool]:
"""(count or None, refused). Only a 500 is a template refusal; 503 = loading or no slot."""
try:
@@ -41634,6 +41708,7 @@ class LlamaCppBackend:
if count is not None:
self._empty_chat_render_refused = True
if count is not None:
+ _count_memo_put(self, memo_key, count)
return count
# The fallback drops role markers, special tokens and tool schemas (~30% of a
diff --git a/studio/backend/tests/test_llama_count_memo.py b/studio/backend/tests/test_llama_count_memo.py
new file mode 100644
index 00000000000..11a83d4d6ae
--- /dev/null
+++ b/studio/backend/tests/test_llama_count_memo.py
@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A tool round near the window recounts the prompt it just fitted; one llama-server render serves both."""
+
+import json
+from types import MethodType, SimpleNamespace
+
+import pytest
+
+
+def _setup(monkeypatch, *, status = 200):
+ from core.inference import llama_cpp
+
+ httpx = llama_cpp.httpx
+ paths = []
+
+ def respond(request):
+ paths.append(request.url.path)
+ if request.url.path == "/apply-template":
+ if status != 200:
+ return httpx.Response(status)
+ body = json.loads(request.content)
+ return httpx.Response(200, json = {"prompt": json.dumps(body)})
+ if request.url.path == "/tokenize":
+ text = json.loads(request.content)["content"]
+ return httpx.Response(200, json = {"tokens": list(range(len(text) // 4))})
+ return httpx.Response(404)
+
+ client_type = httpx.Client
+ monkeypatch.setattr(
+ httpx, "Client", lambda **kw: client_type(transport = httpx.MockTransport(respond), **kw)
+ )
+ backend = SimpleNamespace(
+ base_url = "http://llama.test",
+ is_loaded = True,
+ markup_profile = None,
+ _auth_headers = None,
+ _empty_chat_render_refused = False,
+ _process = SimpleNamespace(pid = 4242),
+ _model_identifier = "org/model-GGUF",
+ )
+ backend.count_chat_tokens = MethodType(llama_cpp.LlamaCppBackend.count_chat_tokens, backend)
+ return backend, paths
+
+
+_MESSAGES = [{"role": "user", "content": "hello " * 200}]
+_TOOLS = [{"type": "function", "function": {"name": "lookup", "parameters": {}}}]
+
+
+def test_identical_recount_reuses_the_render(monkeypatch):
+ backend, paths = _setup(monkeypatch)
+ first = backend.count_chat_tokens(_MESSAGES, None, _TOOLS, strict = True)
+ second = backend.count_chat_tokens(
+ [dict(m) for m in _MESSAGES], None, list(_TOOLS), strict = True
+ )
+ assert first == second > 0
+ assert paths == ["/apply-template", "/tokenize"]
+
+
+ "change",
+ ["messages", "tools", "system", "kwargs", "continue", "options", "process", "model"],
+)
+def test_anything_the_render_sees_misses(monkeypatch, change):
+ backend, paths = _setup(monkeypatch)
+ backend.count_chat_tokens(_MESSAGES, None, _TOOLS, strict = True)
+ messages, system, tools, kwargs = _MESSAGES, None, _TOOLS, {}
+ if change == "messages":
+ messages = _MESSAGES + [{"role": "assistant", "content": "x"}]
+ elif change == "tools":
+ tools = None
+ elif change == "system":
+ system = "Be brief."
+ elif change == "kwargs":
+ kwargs["chat_template_kwargs"] = {"enable_thinking": False}
+ elif change == "continue":
+ messages = _MESSAGES + [{"role": "assistant", "content": "Partial"}]
+ backend.count_chat_tokens(messages, None, tools, strict = True)
+ kwargs["continue_final_message"] = True
+ elif change == "options":
+ kwargs["template_options"] = {"tool_choice": "required"}
+ elif change == "process":
+ backend._process = SimpleNamespace(pid = 4343)
+ elif change == "model":
+ backend._model_identifier = "org/other-GGUF"
+ before = paths.count("/tokenize")
+ backend.count_chat_tokens(messages, system, tools, strict = True, **kwargs)
+ assert paths.count("/tokenize") == before + 1
+
+
+def test_a_failed_render_is_not_remembered(monkeypatch):
+ backend, paths = _setup(monkeypatch, status = 503)
+ with pytest.raises(RuntimeError):
+ backend.count_chat_tokens(_MESSAGES, None, _TOOLS, strict = True)
+ # The non-strict text fallback is an approximation and must not be served later as a count.
+ assert backend.count_chat_tokens(_MESSAGES, None, _TOOLS) > 0
+ before = paths.count("/apply-template")
+ backend.count_chat_tokens(_MESSAGES, None, _TOOLS)
+ assert paths.count("/apply-template") == before + 1
+
+
+def test_admission_counts_are_not_memoised(monkeypatch):
+ backend, paths = _setup(monkeypatch)
+ for _ in range(2):
+ backend.count_chat_tokens(_MESSAGES, None, _TOOLS, strict = True, prefer_native = True)
+ assert paths.count("/v1/chat/completions/input_tokens") == 2
+
+
+def test_memo_stays_bounded(monkeypatch):
+ from core.inference import llama_cpp
+
+ backend, _ = _setup(monkeypatch)
+ for i in range(3 * llama_cpp._COUNT_MEMO_SIZE):
+ backend.count_chat_tokens([{"role": "user", "content": f"turn {i}"}], strict = True)
+ assert len(llama_cpp._count_memo(backend)[1]) == llama_cpp._COUNT_MEMO_SIZE