Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio RAG: batch llama-server token counts and encode the auto-inject query once (#16812)
Daniel Han070e482c76cc
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 14f6ad57264..7dd8eb7b7f9 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -14229,6 +14229,16 @@ def _thread_document_ids(thread_id) -> set | None:def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> dict | None:"""Pre-retrieves the latest turn for models that skip search_knowledge_base. Toggle rag_scope.autoinject or RAG_AUTOINJECT; floor RAG_AUTOINJECT_MIN_SCORE."""+ try:+ from core.rag.retrieval import query_vector_memo+ except Exception: # noqa: BLE001 - the build reports an unavailable RAG stack itself+ return _build_rag_autoinject(conversation, rag_scope)+ # The thread retry, the hybrid retry and the dense probe re-ask the same query: encode it once.+ with query_vector_memo():+ return _build_rag_autoinject(conversation, rag_scope)+++def _build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> dict | None:enabled, whole_doc_requested = rag_autoinject_reaches_retrieval(conversation, rag_scope)if not enabled and not whole_doc_requested:return Nonediff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.pyindex d00a30c950d..32078bfa714 100644--- a/studio/backend/core/rag/embed_llama_server.py+++ b/studio/backend/core/rag/embed_llama_server.py@@ -19,6 +19,7 @@ import subprocessimport sysimport threadingimport time+from collections import OrderedDictfrom contextlib import contextmanagerfrom functools import lru_cachefrom pathlib import Path@@ -53,6 +54,9 @@ _GGUF_SCALAR_WIDTHS = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11# Matches llama.cpp's -ub default; a longer embedding prompt is refused with a 500._UBATCH_SIZE = 512+_TOKEN_COUNT_CACHE_SIZE = 4096+_TOKENIZE_BATCH = 256+_TOKENIZE_BATCH_CHARS = 262144def _skip_gguf_value(f, vtype: int) -> None:@@ -1248,17 +1252,99 @@ class LlamaServerBackend:self.dim(model_name = model_name)def token_counter(self, *, model_name = None):- """Count tokens via the GGUF's /tokenize so chunk sizing matches the- embedder. Cached per text."""+ """Count tokens via the GGUF's /tokenize, cached per text. ``many`` sends a content array:+ tokenize_mixed tokenizes each string alone, so -1 markers split the reply per text."""+ memo: OrderedDict[str, int] = OrderedDict()+ memo_lock = threading.Lock()+ batching = True++ def _remember(text: str, n: int) -> None:+ with memo_lock:+ memo[text] = n+ memo.move_to_end(text)+ if len(memo) > _TOKEN_COUNT_CACHE_SIZE:+ memo.popitem(last = False)++ def _lookup(text: str) -> int | None:+ with memo_lock:+ n = memo.get(text)+ if n is not None:+ memo.move_to_end(text)+ return n++ def _count_one(text: str) -> int:+ data = self._post(+ "/tokenize",+ {"content": text, "add_special": False},+ model_name = model_name,+ )+ return len(data.get("tokens", []))- @lru_cache(maxsize = 4096)def _count(text: str) -> int:- with self._operation():+ n = _lookup(text)+ if n is None:+ n = _count_one(text)+ _remember(text, n)+ return n++ def _count_batch(keys: list[str]) -> list[int] | None:+ try:data = self._post("/tokenize",- {"content": text, "add_special": False},+ {"content": [x for key in keys for x in (-1, key)][1:], "add_special": False},model_name = model_name,)- return len(data.get("tokens", []))-+ except RuntimeError as e:+ # A build that refuses the array answers with a status; anything else still raises.+ if not isinstance(e.__cause__, httpx.HTTPStatusError):+ raise+ return None+ counts, run = [], 0+ for token in data.get("tokens", []):+ if token == -1:+ counts.append(run)+ run = 0+ else:+ run += 1+ counts.append(run)+ return counts if len(counts) == len(keys) else None++ def _count_many(texts) -> list[int]:+ nonlocal batching+ texts = list(texts)+ out = [0] * len(texts)+ misses: dict[str, list[int]] = {}+ for i, text in enumerate(texts):+ n = _lookup(text)+ if n is None:+ misses.setdefault(text, []).append(i)+ else:+ out[i] = n+ keys = list(misses)+ start = 0+ while start < len(keys):+ # Bounded batches, so a query encode waiting on _serve_lock never waits out a whole document.+ end, chars = start, 0+ while (+ end < len(keys)+ and end - start < _TOKENIZE_BATCH+ and (end == start or chars + len(keys[end]) <= _TOKENIZE_BATCH_CHARS)+ ):+ chars += len(keys[end])+ end += 1+ batch = keys[start:end]+ counts = None+ if batching and len(batch) > 1:+ counts = _count_batch(batch)+ batching = counts is not None+ if counts is None:+ counts = [_count_one(key) for key in batch]+ for key, n in zip(batch, counts):+ _remember(key, n)+ for i in misses[key]:+ out[i] = n+ start = end+ return out++ _count.many = _count_manyreturn _countdiff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex c64d433e31e..41ff05ae622 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -1530,7 +1530,7 @@ def token_counter(model_name: str | None = None) -> Callable[[str], int]:texts = list(texts)return _served(lambda count: _many(count, texts))- # Only a batching backend (sentence-transformers) changes how chunking asks for counts.+ # Only a batching backend (sentence-transformers, llama-server) changes how chunking asks for counts.if getattr(state[1], "many", None) is not None:_count.many = _count_manyreturn _countdiff --git a/studio/backend/tests/test_rag_llama_token_batch.py b/studio/backend/tests/test_rag_llama_token_batch.pynew file mode 100644index 00000000000..1832cac796a--- /dev/null+++ b/studio/backend/tests/test_rag_llama_token_batch.py@@ -0,0 +1,223 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""The llama-server token counter counts many texts per /tokenize POST, and auto-inject encodes its query once."""++import json+import math+import os+import sys++import httpx+import pytest++sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from core.rag import embeddings, store # noqa: E402+from core.rag.chunking import Chunk, Page, chunk_pages # noqa: E402+from core.rag.embed_llama_server import LlamaServerBackend # noqa: E402+++def _tokens(text):+ # Stand-in tokenizer: one token per word and per punctuation mark, never negative ids.+ return [hash(w) & 0xFFFF for w in text.replace(".", " . ").replace(",", " , ").split()]+++class _Server:+ """Answers /tokenize the way llama-server's tokenize_mixed does: strings tokenized alone, ints kept."""++ def __init__(+ self,+ refuse_arrays = False,+ drop_markers = False,+ ):+ self.posts = []+ self.refuse_arrays = refuse_arrays+ self.drop_markers = drop_markers++ def __call__(self, path, payload, **_kwargs):+ assert path == "/tokenize" and payload["add_special"] is False+ content = payload["content"]+ self.posts.append(content)+ if isinstance(content, str):+ return {"tokens": _tokens(content)}+ if self.refuse_arrays:+ request = httpx.Request("POST", "http://x/tokenize")+ response = httpx.Response(400, request = request, text = "bad")+ try:+ response.raise_for_status()+ except httpx.HTTPStatusError as e:+ raise RuntimeError("llama-server embedder POST /tokenize -> 400: bad") from e+ out = []+ for part in content:+ if isinstance(part, str):+ out.extend(_tokens(part))+ elif not self.drop_markers:+ out.append(part)+ return {"tokens": out}+++def _counter(monkeypatch, server):+ b = LlamaServerBackend()+ monkeypatch.setattr(b, "_post", server)+ return b.token_counter()+++def _pages(n = 12):+ para = "Alpha bravo, charlie delta. Echo foxtrot golf hotel india juliet kilo lima.\n\n"+ return [+ Page(text = (para * (3 + i % 5)) + "x " * (40 * (i % 3)), page_number = i + 1, char_count = 0)+ for i in range(n)+ ]+++def _rows(chunks):+ return [+ (+ c.text,+ c.token_count,+ c.page_number,+ c.source_page_index,+ c.chunk_index,+ c.page_char_start,+ c.page_char_end,+ c.whole_document_token_count,+ )+ for c in chunks+ ]+++def test_batched_counts_equal_single_counts(monkeypatch):+ server = _Server()+ count = _counter(monkeypatch, server)+ texts = ["a b c", "", "d, e.", "a b c", "one", " "]+ assert count.many(texts) == [len(_tokens(t)) for t in texts]+ # One array POST for the five distinct texts, then the memo answers singles and repeats.+ assert len(server.posts) == 1 and server.posts[0][1] == -1+ assert count("d, e.") == 4 and count.many(["one", "a b c"]) == [1, 3]+ assert len(server.posts) == 1+++def test_lone_text_and_singles_keep_the_string_request(monkeypatch):+ server = _Server()+ count = _counter(monkeypatch, server)+ assert count.many(["hello world"]) == [2]+ assert count("x y z") == 3+ assert server.posts == ["hello world", "x y z"]++[email protected]("max_tokens,overlap", [(8, 0), (16, 4), (64, 16), (512, 64)])+def test_chunks_and_posts_vs_one_text_per_post(monkeypatch, max_tokens, overlap):+ single = _Server()+ plain = _counter(monkeypatch, single)+ one_at_a_time = lambda text: plain(text) # noqa: E731 - no batch form: the old path+ batched_server = _Server()+ batched = _counter(monkeypatch, batched_server)+ pages = _pages()+ expected = _rows(+ chunk_pages(pages, max_tokens = max_tokens, overlap = overlap, count = one_at_a_time)+ )+ assert (+ _rows(chunk_pages(pages, max_tokens = max_tokens, overlap = overlap, count = batched)) == expected+ )+ assert len(batched_server.posts) < len(single.posts) / 4++[email protected]("server", [_Server(refuse_arrays = True), _Server(drop_markers = True)])+def test_a_server_that_cannot_batch_falls_back_to_single_posts(monkeypatch, server):+ count = _counter(monkeypatch, server)+ texts = ["a b", "c d e", "f"]+ assert count.many(texts) == [2, 3, 1]+ assert server.posts[1:] == texts+ assert count.many(["g h", "i"]) == [2, 1]+ assert server.posts[-2:] == ["g h", "i"]+++def test_a_failed_batch_post_still_raises(monkeypatch):+ def broken(path, payload, **_kwargs):+ raise RuntimeError("llama-server embedder POST /tokenize failed after retry")++ count = _counter(monkeypatch, broken)+ with pytest.raises(RuntimeError, match = "failed after retry"):+ count.many(["a", "b"])+++def test_batches_are_bounded(monkeypatch):+ from core.rag import embed_llama_server as mod++ monkeypatch.setattr(mod, "_TOKENIZE_BATCH", 3)+ monkeypatch.setattr(mod, "_TOKENIZE_BATCH_CHARS", 10)+ server = _Server()+ count = _counter(monkeypatch, server)+ texts = [f"w{i}" for i in range(7)] + ["a long text of many chars"]+ assert count.many(texts) == [1] * 7 + [6]+ sizes = [1 if isinstance(p, str) else (len(p) + 1) // 2 for p in server.posts]+ assert sizes == [3, 3, 1, 1]+++def test_shared_token_counter_exposes_the_batch_form(monkeypatch):+ server = _Server()+ b = LlamaServerBackend()+ monkeypatch.setattr(b, "_post", server)+ monkeypatch.setattr(embeddings, "_get_backend", lambda model_name = None: b)+ count = embeddings.token_counter("org/embedder-GGUF")+ assert count.many(["a b", "c"]) == [2, 1] and len(server.posts) == 1+++VOCAB = ["alpha", "bravo", "summary", "thread"]+++def _embed(text):+ v = [float(text.lower().count(w)) + 0.01 for w in VOCAB]+ n = math.sqrt(sum(x * x for x in v))+ return [x / n for x in v]+++def test_autoinject_encodes_its_query_once(rag_conn, monkeypatch):+ from core.inference import tools+ from storage import rag_db++ calls = []++ def encode(+ texts,+ *,+ model_name = None,+ normalize = True,+ ):+ calls.append(list(texts))+ return [_embed(t) for t in texts]++ monkeypatch.setattr(embeddings, "encode", encode)+ monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(VOCAB))+ monkeypatch.setattr(rag_db, "RAG_AVAILABLE", True, raising = False)+ monkeypatch.setattr(rag_db, "rag_available", lambda: True, raising = False)+ text = "alpha bravo thread notes"+ store.create_document(+ rag_conn, scope = "thread_T", filename = "t.pdf", sha256 = "dt", document_id = "dt"+ )+ chunk = Chunk(+ text = text,+ token_count = 4,+ page_number = None,+ source_page_index = 0,+ chunk_index = 0,+ page_char_start = 0,+ page_char_end = len(text),+ )+ store.add_chunks(rag_conn, "thread_T", "dt", [chunk], [_embed(text)])+ rag_conn.execute("UPDATE documents SET status='completed', num_chunks=1 WHERE id='dt'")+ rag_conn.commit()+ # The floored search finds nothing strong, so the unfloored thread search asks the same query again.+ out = tools.build_rag_autoinject(+ [{"role": "user", "content": "summarize this"}],+ {+ "thread_id": "T",+ "autoinject": True,+ "autoinject_min_score": 0.99,+ "mode": "hybrid",+ "context_length": 100,+ },+ )+ assert out is not None and "t.pdf" in json.dumps(out)+ assert calls == [["summarize this"]]