Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio RAG: batch llama-server token counts and encode the auto-inject query once (#16812)

Daniel Han070e482c76cc
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 14f6ad57264..7dd8eb7b7f9 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -14229,6 +14229,16 @@ def _thread_document_ids(thread_id) -> set | None:
def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> dict | None:
"""Pre-retrieves the latest turn for models that skip search_knowledge_base. Toggle rag_scope.autoinject or RAG_AUTOINJECT; floor RAG_AUTOINJECT_MIN_SCORE."""
+ try:
+ from core.rag.retrieval import query_vector_memo
+ except Exception: # noqa: BLE001 - the build reports an unavailable RAG stack itself
+ return _build_rag_autoinject(conversation, rag_scope)
+ # The thread retry, the hybrid retry and the dense probe re-ask the same query: encode it once.
+ with query_vector_memo():
+ return _build_rag_autoinject(conversation, rag_scope)
+
+
+def _build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> dict | None:
enabled, whole_doc_requested = rag_autoinject_reaches_retrieval(conversation, rag_scope)
if not enabled and not whole_doc_requested:
return None
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index d00a30c950d..32078bfa714 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -19,6 +19,7 @@ import subprocess
import sys
import threading
import time
+from collections import OrderedDict
from contextlib import contextmanager
from functools import lru_cache
from pathlib import Path
@@ -53,6 +54,9 @@ _GGUF_SCALAR_WIDTHS = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1, 10: 8, 11
# Matches llama.cpp's -ub default; a longer embedding prompt is refused with a 500.
_UBATCH_SIZE = 512
+_TOKEN_COUNT_CACHE_SIZE = 4096
+_TOKENIZE_BATCH = 256
+_TOKENIZE_BATCH_CHARS = 262144
def _skip_gguf_value(f, vtype: int) -> None:
@@ -1248,17 +1252,99 @@ class LlamaServerBackend:
self.dim(model_name = model_name)
def token_counter(self, *, model_name = None):
- """Count tokens via the GGUF's /tokenize so chunk sizing matches the
- embedder. Cached per text."""
+ """Count tokens via the GGUF's /tokenize, cached per text. ``many`` sends a content array:
+ tokenize_mixed tokenizes each string alone, so -1 markers split the reply per text."""
+ memo: OrderedDict[str, int] = OrderedDict()
+ memo_lock = threading.Lock()
+ batching = True
+
+ def _remember(text: str, n: int) -> None:
+ with memo_lock:
+ memo[text] = n
+ memo.move_to_end(text)
+ if len(memo) > _TOKEN_COUNT_CACHE_SIZE:
+ memo.popitem(last = False)
+
+ def _lookup(text: str) -> int | None:
+ with memo_lock:
+ n = memo.get(text)
+ if n is not None:
+ memo.move_to_end(text)
+ return n
+
+ def _count_one(text: str) -> int:
+ data = self._post(
+ "/tokenize",
+ {"content": text, "add_special": False},
+ model_name = model_name,
+ )
+ return len(data.get("tokens", []))
- @lru_cache(maxsize = 4096)
def _count(text: str) -> int:
- with self._operation():
+ n = _lookup(text)
+ if n is None:
+ n = _count_one(text)
+ _remember(text, n)
+ return n
+
+ def _count_batch(keys: list[str]) -> list[int] | None:
+ try:
data = self._post(
"/tokenize",
- {"content": text, "add_special": False},
+ {"content": [x for key in keys for x in (-1, key)][1:], "add_special": False},
model_name = model_name,
)
- return len(data.get("tokens", []))
-
+ except RuntimeError as e:
+ # A build that refuses the array answers with a status; anything else still raises.
+ if not isinstance(e.__cause__, httpx.HTTPStatusError):
+ raise
+ return None
+ counts, run = [], 0
+ for token in data.get("tokens", []):
+ if token == -1:
+ counts.append(run)
+ run = 0
+ else:
+ run += 1
+ counts.append(run)
+ return counts if len(counts) == len(keys) else None
+
+ def _count_many(texts) -> list[int]:
+ nonlocal batching
+ texts = list(texts)
+ out = [0] * len(texts)
+ misses: dict[str, list[int]] = {}
+ for i, text in enumerate(texts):
+ n = _lookup(text)
+ if n is None:
+ misses.setdefault(text, []).append(i)
+ else:
+ out[i] = n
+ keys = list(misses)
+ start = 0
+ while start < len(keys):
+ # Bounded batches, so a query encode waiting on _serve_lock never waits out a whole document.
+ end, chars = start, 0
+ while (
+ end < len(keys)
+ and end - start < _TOKENIZE_BATCH
+ and (end == start or chars + len(keys[end]) <= _TOKENIZE_BATCH_CHARS)
+ ):
+ chars += len(keys[end])
+ end += 1
+ batch = keys[start:end]
+ counts = None
+ if batching and len(batch) > 1:
+ counts = _count_batch(batch)
+ batching = counts is not None
+ if counts is None:
+ counts = [_count_one(key) for key in batch]
+ for key, n in zip(batch, counts):
+ _remember(key, n)
+ for i in misses[key]:
+ out[i] = n
+ start = end
+ return out
+
+ _count.many = _count_many
return _count
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index c64d433e31e..41ff05ae622 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -1530,7 +1530,7 @@ def token_counter(model_name: str | None = None) -> Callable[[str], int]:
texts = list(texts)
return _served(lambda count: _many(count, texts))
- # Only a batching backend (sentence-transformers) changes how chunking asks for counts.
+ # Only a batching backend (sentence-transformers, llama-server) changes how chunking asks for counts.
if getattr(state[1], "many", None) is not None:
_count.many = _count_many
return _count
diff --git a/studio/backend/tests/test_rag_llama_token_batch.py b/studio/backend/tests/test_rag_llama_token_batch.py
new file mode 100644
index 00000000000..1832cac796a
--- /dev/null
+++ b/studio/backend/tests/test_rag_llama_token_batch.py
@@ -0,0 +1,223 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The llama-server token counter counts many texts per /tokenize POST, and auto-inject encodes its query once."""
+
+import json
+import math
+import os
+import sys
+
+import httpx
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from core.rag import embeddings, store # noqa: E402
+from core.rag.chunking import Chunk, Page, chunk_pages # noqa: E402
+from core.rag.embed_llama_server import LlamaServerBackend # noqa: E402
+
+
+def _tokens(text):
+ # Stand-in tokenizer: one token per word and per punctuation mark, never negative ids.
+ return [hash(w) & 0xFFFF for w in text.replace(".", " . ").replace(",", " , ").split()]
+
+
+class _Server:
+ """Answers /tokenize the way llama-server's tokenize_mixed does: strings tokenized alone, ints kept."""
+
+ def __init__(
+ self,
+ refuse_arrays = False,
+ drop_markers = False,
+ ):
+ self.posts = []
+ self.refuse_arrays = refuse_arrays
+ self.drop_markers = drop_markers
+
+ def __call__(self, path, payload, **_kwargs):
+ assert path == "/tokenize" and payload["add_special"] is False
+ content = payload["content"]
+ self.posts.append(content)
+ if isinstance(content, str):
+ return {"tokens": _tokens(content)}
+ if self.refuse_arrays:
+ request = httpx.Request("POST", "http://x/tokenize")
+ response = httpx.Response(400, request = request, text = "bad")
+ try:
+ response.raise_for_status()
+ except httpx.HTTPStatusError as e:
+ raise RuntimeError("llama-server embedder POST /tokenize -> 400: bad") from e
+ out = []
+ for part in content:
+ if isinstance(part, str):
+ out.extend(_tokens(part))
+ elif not self.drop_markers:
+ out.append(part)
+ return {"tokens": out}
+
+
+def _counter(monkeypatch, server):
+ b = LlamaServerBackend()
+ monkeypatch.setattr(b, "_post", server)
+ return b.token_counter()
+
+
+def _pages(n = 12):
+ para = "Alpha bravo, charlie delta. Echo foxtrot golf hotel india juliet kilo lima.\n\n"
+ return [
+ Page(text = (para * (3 + i % 5)) + "x " * (40 * (i % 3)), page_number = i + 1, char_count = 0)
+ for i in range(n)
+ ]
+
+
+def _rows(chunks):
+ return [
+ (
+ c.text,
+ c.token_count,
+ c.page_number,
+ c.source_page_index,
+ c.chunk_index,
+ c.page_char_start,
+ c.page_char_end,
+ c.whole_document_token_count,
+ )
+ for c in chunks
+ ]
+
+
+def test_batched_counts_equal_single_counts(monkeypatch):
+ server = _Server()
+ count = _counter(monkeypatch, server)
+ texts = ["a b c", "", "d, e.", "a b c", "one", " "]
+ assert count.many(texts) == [len(_tokens(t)) for t in texts]
+ # One array POST for the five distinct texts, then the memo answers singles and repeats.
+ assert len(server.posts) == 1 and server.posts[0][1] == -1
+ assert count("d, e.") == 4 and count.many(["one", "a b c"]) == [1, 3]
+ assert len(server.posts) == 1
+
+
+def test_lone_text_and_singles_keep_the_string_request(monkeypatch):
+ server = _Server()
+ count = _counter(monkeypatch, server)
+ assert count.many(["hello world"]) == [2]
+ assert count("x y z") == 3
+ assert server.posts == ["hello world", "x y z"]
+
+
[email protected]("max_tokens,overlap", [(8, 0), (16, 4), (64, 16), (512, 64)])
+def test_chunks_and_posts_vs_one_text_per_post(monkeypatch, max_tokens, overlap):
+ single = _Server()
+ plain = _counter(monkeypatch, single)
+ one_at_a_time = lambda text: plain(text) # noqa: E731 - no batch form: the old path
+ batched_server = _Server()
+ batched = _counter(monkeypatch, batched_server)
+ pages = _pages()
+ expected = _rows(
+ chunk_pages(pages, max_tokens = max_tokens, overlap = overlap, count = one_at_a_time)
+ )
+ assert (
+ _rows(chunk_pages(pages, max_tokens = max_tokens, overlap = overlap, count = batched)) == expected
+ )
+ assert len(batched_server.posts) < len(single.posts) / 4
+
+
[email protected]("server", [_Server(refuse_arrays = True), _Server(drop_markers = True)])
+def test_a_server_that_cannot_batch_falls_back_to_single_posts(monkeypatch, server):
+ count = _counter(monkeypatch, server)
+ texts = ["a b", "c d e", "f"]
+ assert count.many(texts) == [2, 3, 1]
+ assert server.posts[1:] == texts
+ assert count.many(["g h", "i"]) == [2, 1]
+ assert server.posts[-2:] == ["g h", "i"]
+
+
+def test_a_failed_batch_post_still_raises(monkeypatch):
+ def broken(path, payload, **_kwargs):
+ raise RuntimeError("llama-server embedder POST /tokenize failed after retry")
+
+ count = _counter(monkeypatch, broken)
+ with pytest.raises(RuntimeError, match = "failed after retry"):
+ count.many(["a", "b"])
+
+
+def test_batches_are_bounded(monkeypatch):
+ from core.rag import embed_llama_server as mod
+
+ monkeypatch.setattr(mod, "_TOKENIZE_BATCH", 3)
+ monkeypatch.setattr(mod, "_TOKENIZE_BATCH_CHARS", 10)
+ server = _Server()
+ count = _counter(monkeypatch, server)
+ texts = [f"w{i}" for i in range(7)] + ["a long text of many chars"]
+ assert count.many(texts) == [1] * 7 + [6]
+ sizes = [1 if isinstance(p, str) else (len(p) + 1) // 2 for p in server.posts]
+ assert sizes == [3, 3, 1, 1]
+
+
+def test_shared_token_counter_exposes_the_batch_form(monkeypatch):
+ server = _Server()
+ b = LlamaServerBackend()
+ monkeypatch.setattr(b, "_post", server)
+ monkeypatch.setattr(embeddings, "_get_backend", lambda model_name = None: b)
+ count = embeddings.token_counter("org/embedder-GGUF")
+ assert count.many(["a b", "c"]) == [2, 1] and len(server.posts) == 1
+
+
+VOCAB = ["alpha", "bravo", "summary", "thread"]
+
+
+def _embed(text):
+ v = [float(text.lower().count(w)) + 0.01 for w in VOCAB]
+ n = math.sqrt(sum(x * x for x in v))
+ return [x / n for x in v]
+
+
+def test_autoinject_encodes_its_query_once(rag_conn, monkeypatch):
+ from core.inference import tools
+ from storage import rag_db
+
+ calls = []
+
+ def encode(
+ texts,
+ *,
+ model_name = None,
+ normalize = True,
+ ):
+ calls.append(list(texts))
+ return [_embed(t) for t in texts]
+
+ monkeypatch.setattr(embeddings, "encode", encode)
+ monkeypatch.setattr(embeddings, "dim", lambda model_name = None: len(VOCAB))
+ monkeypatch.setattr(rag_db, "RAG_AVAILABLE", True, raising = False)
+ monkeypatch.setattr(rag_db, "rag_available", lambda: True, raising = False)
+ text = "alpha bravo thread notes"
+ store.create_document(
+ rag_conn, scope = "thread_T", filename = "t.pdf", sha256 = "dt", document_id = "dt"
+ )
+ chunk = Chunk(
+ text = text,
+ token_count = 4,
+ page_number = None,
+ source_page_index = 0,
+ chunk_index = 0,
+ page_char_start = 0,
+ page_char_end = len(text),
+ )
+ store.add_chunks(rag_conn, "thread_T", "dt", [chunk], [_embed(text)])
+ rag_conn.execute("UPDATE documents SET status='completed', num_chunks=1 WHERE id='dt'")
+ rag_conn.commit()
+ # The floored search finds nothing strong, so the unfloored thread search asks the same query again.
+ out = tools.build_rag_autoinject(
+ [{"role": "user", "content": "summarize this"}],
+ {
+ "thread_id": "T",
+ "autoinject": True,
+ "autoinject_min_score": 0.99,
+ "mode": "hybrid",
+ "context_length": 100,
+ },
+ )
+ assert out is not None and "t.pdf" in json.dumps(out)
+ assert calls == [["summarize this"]]