Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: tokenize a repeated chunk text once in the sentence-transformers counter (#15664)
Daniel Han04a993739da8
diff --git a/studio/backend/core/rag/embeddings.py b/studio/backend/core/rag/embeddings.pyindex dcaf00919af..8c31ec08aca 100644--- a/studio/backend/core/rag/embeddings.py+++ b/studio/backend/core/rag/embeddings.py@@ -12,6 +12,7 @@ import loggingimport osimport reimport threading+import weakreffrom contextlib import contextmanagerfrom contextvars import ContextVarfrom functools import lru_cache@@ -720,17 +721,42 @@ def _st_max_tokens(model_name: str | None = None) -> int | None:return max(1, int(limit) - reserved)+_ST_COUNT_CACHE_SIZE = 4096++def _st_token_counter(model_name: str | None = None) -> Callable[[str], int]:- """Reads the tokenizer per call under _compute_lock; caching it would outlive an unload."""+ """Reads the tokenizer per call under _compute_lock; caching it would outlive an unload.+ The count memo is tied to that tokenizer by a weakref: a switch empties it, an unload frees it."""+ counts: dict[str, int] = {}+ counted_by: list = [None]def _count(t: str) -> int:with _compute_lock:tok = _get(model_name).tokenizer+ owner = counted_by[0]+ if owner is None or owner() is not tok:+ counts.clear()+ try:+ counted_by[0] = weakref.ref(tok)+ except TypeError:+ counted_by[0] = None+ memo = counted_by[0] is not None and type(t) is str+ known = counts.get(t) if memo else None+ if known is not None:+ # Leave the environment a tokenize would.+ if os.environ.get("TOKENIZERS_PARALLELISM") != "false":+ os.environ["TOKENIZERS_PARALLELISM"] = "false"+ return knownos.environ["TOKENIZERS_PARALLELISM"] = "true"try:- return len(tok.encode(t, add_special_tokens = False))+ n = len(tok.encode(t, add_special_tokens = False))finally:os.environ["TOKENIZERS_PARALLELISM"] = "false"+ if memo:+ if len(counts) >= _ST_COUNT_CACHE_SIZE:+ counts.pop(next(iter(counts)))+ counts[t] = n+ return nreturn _countdiff --git a/studio/backend/tests/test_rag_st_token_count_memo.py b/studio/backend/tests/test_rag_st_token_count_memo.pynew file mode 100644index 00000000000..8659ce461b8--- /dev/null+++ b/studio/backend/tests/test_rag_st_token_count_memo.py@@ -0,0 +1,130 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import gc+import os+import weakref++from core.rag import embeddings+from core.rag.chunking import chunk_pages+from core.rag.parsers import Page+++class _Tok:+ def __init__(self, per_word = 1):+ self.per_word = per_word+ self.calls = []++ def encode(+ self,+ text,+ add_special_tokens = False,+ ):+ self.calls.append(text)+ return [0] * (self.per_word * len(text.split()) + len(text) % 3)+++class _Model:+ def __init__(self, tok):+ self.tokenizer = tok+++def test_a_repeated_text_is_tokenized_once(monkeypatch):+ tok = _Tok()+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _Model(tok))+ count = embeddings._st_token_counter("org/embedder")+ assert [count("a b c"), count("a b c"), count("d e"), count("a b c")] == [5, 5, 2, 5]+ assert tok.calls == ["a b c", "d e"]+++def test_switching_the_model_between_calls_recounts(monkeypatch):+ """model_name=None follows the configured model on every call, so a count memoized+ for the old tokenizer must not answer for the new one."""+ old, new = _Tok(1), _Tok(5)+ models = {"current": _Model(old)}+ looked_up = []++ def _get(model_name = None):+ looked_up.append(model_name)+ return models["current"]++ monkeypatch.setattr(embeddings, "_get", _get)+ count = embeddings._st_token_counter(None)+ assert count("one two three") == 3 + len("one two three") % 3+ models["current"] = _Model(new)+ assert count("one two three") == 15 + len("one two three") % 3+ models["current"] = _Model(old)+ assert count("one two three") == 3 + len("one two three") % 3+ assert len(old.calls) == 2 and len(new.calls) == 1+ assert looked_up == [None, None, None]+++def test_the_memo_does_not_keep_an_unloaded_tokenizer_alive(monkeypatch):+ holder = {"model": _Model(_Tok())}+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: holder["model"])+ count = embeddings._st_token_counter("org/embedder")+ count("alpha beta")+ gone = weakref.ref(holder["model"].tokenizer)+ holder["model"] = _Model(_Tok())+ gc.collect()+ assert gone() is None+ assert count("alpha beta") == 2 + len("alpha beta") % 3+++def test_a_memoized_count_leaves_the_same_environment(monkeypatch):+ tok = _Tok()+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _Model(tok))+ monkeypatch.setenv("TOKENIZERS_PARALLELISM", "false")+ count = embeddings._st_token_counter("org/embedder")+ count("x y")+ os.environ["TOKENIZERS_PARALLELISM"] = "true"+ assert count("x y") == 2 + len("x y") % 3+ assert os.environ["TOKENIZERS_PARALLELISM"] == "false"+ assert len(tok.calls) == 1+++def test_a_tokenizer_without_weakref_support_is_counted_every_time(monkeypatch):+ class _SlotTok:+ __slots__ = ("calls",)++ def __init__(self):+ self.calls = 0++ def encode(+ self,+ text,+ add_special_tokens = False,+ ):+ self.calls += 1+ return [0] * len(text.split())++ tok = _SlotTok()+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _Model(tok))+ count = embeddings._st_token_counter("org/embedder")+ assert [count("a b"), count("a b")] == [2, 2]+ assert tok.calls == 2+++def test_chunks_are_unchanged_and_repeats_are_not_retokenized(monkeypatch):+ """Text with no spaces (as in CJK) is split per character, repeating the same pieces."""++ class _CharTok(_Tok):+ def encode(+ self,+ text,+ add_special_tokens = False,+ ):+ self.calls.append(text)+ return [0] * len(text.strip())++ tok = _CharTok()+ monkeypatch.setattr(embeddings, "_get", lambda model_name = None: _Model(tok))+ text = ("".join(chr(0x4E00 + (i * 7) % 40) for i in range(300)) + "\n") * 3+ pages = [Page(text = text, page_number = 1, char_count = len(text))]+ expected = chunk_pages(+ pages, max_tokens = 48, overlap = 8, count = lambda t: len(_CharTok().encode(t))+ )+ got = chunk_pages(pages, max_tokens = 48, overlap = 8, count = embeddings._st_token_counter("m"))+ assert got == expected+ assert len(got) > 10+ assert len(tok.calls) == len(set(tok.calls)) < 200