Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: tokenize a repeated chunk text once in the sentence-transformers counter (#15664)

Daniel Han04a993739da8
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.py
index dcaf00919af..8c31ec08aca 100644
--- a/studio/backend/core/rag/embeddings.py
+++ b/studio/backend/core/rag/embeddings.py
@@ -12,6 +12,7 @@ import logging
import os
import re
import threading
+import weakref
from contextlib import contextmanager
from contextvars import ContextVar
from functools import lru_cache
@@ -720,17 +721,42 @@ def _st_max_tokens(model_name: str | None = None) -> int | None:
return max(1, int(limit) - reserved)
+_ST_COUNT_CACHE_SIZE = 4096
+
+
def _st_token_counter(model_name: str | None = None) -> Callable[[str], int]:
- """Reads the tokenizer per call under _compute_lock; caching it would outlive an unload."""
+ """Reads the tokenizer per call under _compute_lock; caching it would outlive an unload.
+ The count memo is tied to that tokenizer by a weakref: a switch empties it, an unload frees it."""
+ counts: dict[str, int] = {}
+ counted_by: list = [None]
def _count(t: str) -> int:
with _compute_lock:
tok = _get(model_name).tokenizer
+ owner = counted_by[0]
+ if owner is None or owner() is not tok:
+ counts.clear()
+ try:
+ counted_by[0] = weakref.ref(tok)
+ except TypeError:
+ counted_by[0] = None
+ memo = counted_by[0] is not None and type(t) is str
+ known = counts.get(t) if memo else None
+ if known is not None:
+ # Leave the environment a tokenize would.
+ if os.environ.get("TOKENIZERS_PARALLELISM") != "false":
+ os.environ["TOKENIZERS_PARALLELISM"] = "false"
+ return known
os.environ["TOKENIZERS_PARALLELISM"] = "true"
try:
- return len(tok.encode(t, add_special_tokens = False))
+ n = len(tok.encode(t, add_special_tokens = False))
finally:
os.environ["TOKENIZERS_PARALLELISM"] = "false"
+ if memo:
+ if len(counts) >= _ST_COUNT_CACHE_SIZE:
+ counts.pop(next(iter(counts)))
+ counts[t] = n
+ return n
return _count
diff --git a/studio/backend/tests/test_rag_st_token_count_memo.py b/studio/backend/tests/test_rag_st_token_count_memo.py
new file mode 100644
index 00000000000..8659ce461b8
--- /dev/null
+++ b/studio/backend/tests/test_rag_st_token_count_memo.py
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import gc
+import os
+import weakref
+
+from core.rag import embeddings
+from core.rag.chunking import chunk_pages
+from core.rag.parsers import Page
+
+
+class _Tok:
+ def __init__(self, per_word = 1):
+ self.per_word = per_word
+ self.calls = []
+
+ def encode(
+ self,
+ text,
+ add_special_tokens = False,
+ ):
+ self.calls.append(text)
+ return [0] * (self.per_word * len(text.split()) + len(text) % 3)
+
+
+class _Model:
+ def __init__(self, tok):
+ self.tokenizer = tok
+
+
+def test_a_repeated_text_is_tokenized_once(monkeypatch):
+ tok = _Tok()
+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _Model(tok))
+ count = embeddings._st_token_counter("org/embedder")
+ assert [count("a b c"), count("a b c"), count("d e"), count("a b c")] == [5, 5, 2, 5]
+ assert tok.calls == ["a b c", "d e"]
+
+
+def test_switching_the_model_between_calls_recounts(monkeypatch):
+ """model_name=None follows the configured model on every call, so a count memoized
+ for the old tokenizer must not answer for the new one."""
+ old, new = _Tok(1), _Tok(5)
+ models = {"current": _Model(old)}
+ looked_up = []
+
+ def _get(model_name = None):
+ looked_up.append(model_name)
+ return models["current"]
+
+ monkeypatch.setattr(embeddings, "_get", _get)
+ count = embeddings._st_token_counter(None)
+ assert count("one two three") == 3 + len("one two three") % 3
+ models["current"] = _Model(new)
+ assert count("one two three") == 15 + len("one two three") % 3
+ models["current"] = _Model(old)
+ assert count("one two three") == 3 + len("one two three") % 3
+ assert len(old.calls) == 2 and len(new.calls) == 1
+ assert looked_up == [None, None, None]
+
+
+def test_the_memo_does_not_keep_an_unloaded_tokenizer_alive(monkeypatch):
+ holder = {"model": _Model(_Tok())}
+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: holder["model"])
+ count = embeddings._st_token_counter("org/embedder")
+ count("alpha beta")
+ gone = weakref.ref(holder["model"].tokenizer)
+ holder["model"] = _Model(_Tok())
+ gc.collect()
+ assert gone() is None
+ assert count("alpha beta") == 2 + len("alpha beta") % 3
+
+
+def test_a_memoized_count_leaves_the_same_environment(monkeypatch):
+ tok = _Tok()
+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _Model(tok))
+ monkeypatch.setenv("TOKENIZERS_PARALLELISM", "false")
+ count = embeddings._st_token_counter("org/embedder")
+ count("x y")
+ os.environ["TOKENIZERS_PARALLELISM"] = "true"
+ assert count("x y") == 2 + len("x y") % 3
+ assert os.environ["TOKENIZERS_PARALLELISM"] == "false"
+ assert len(tok.calls) == 1
+
+
+def test_a_tokenizer_without_weakref_support_is_counted_every_time(monkeypatch):
+ class _SlotTok:
+ __slots__ = ("calls",)
+
+ def __init__(self):
+ self.calls = 0
+
+ def encode(
+ self,
+ text,
+ add_special_tokens = False,
+ ):
+ self.calls += 1
+ return [0] * len(text.split())
+
+ tok = _SlotTok()
+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _Model(tok))
+ count = embeddings._st_token_counter("org/embedder")
+ assert [count("a b"), count("a b")] == [2, 2]
+ assert tok.calls == 2
+
+
+def test_chunks_are_unchanged_and_repeats_are_not_retokenized(monkeypatch):
+ """Text with no spaces (as in CJK) is split per character, repeating the same pieces."""
+
+ class _CharTok(_Tok):
+ def encode(
+ self,
+ text,
+ add_special_tokens = False,
+ ):
+ self.calls.append(text)
+ return [0] * len(text.strip())
+
+ tok = _CharTok()
+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _Model(tok))
+ text = ("".join(chr(0x4E00 + (i * 7) % 40) for i in range(300)) + "\n") * 3
+ pages = [Page(text = text, page_number = 1, char_count = len(text))]
+ expected = chunk_pages(
+ pages, max_tokens = 48, overlap = 8, count = lambda t: len(_CharTok().encode(t))
+ )
+ got = chunk_pages(pages, max_tokens = 48, overlap = 8, count = embeddings._st_token_counter("m"))
+ assert got == expected
+ assert len(got) > 10
+ assert len(tok.calls) == len(set(tok.calls)) < 200