Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""``tokenizer.decode(ids)`` for a reply growing one id at a time, decoding only the ids after the
last point where decoding is known to split (HF fast tokenizers with ByteLevel, Metaspace or
per-token Replace / ByteFallback / Fuse / Strip(start <= 1) decoders). A split needs both ids to be
non-added, non-``<0xNN>`` pieces that decode alone to non-empty text without U+FFFD, and, when
cleanup may run, the left one to end in a character no cleanup pattern contains. Anything else,
or an overridden decode, decodes every id as before."""
from __future__ import annotations
import json
import re
import threading
import weakref
__all__ = ["IncrementalDecode", "split_profile"]
# Every character of PreTrainedTokenizerBase.clean_up_tokenization's patterns.
_CLEANUP_CHARS = frozenset(" .?!,'ntmsvre")
_BYTE_PIECE = re.compile(r"<0x[0-9A-Fa-f]{2}>")
_SPLIT_DECODE = {"skip_special_tokens": False, "clean_up_tokenization_spaces": False}
def _decoder_steps(config):
if isinstance(config, dict) and config.get("type") == "Sequence":
steps = config.get("decoders")
if not isinstance(steps, list):
return None
flat = []
for step in steps:
inner = _decoder_steps(step)
if inner is None:
return None
flat.extend(inner)
return flat
return [config]
def _step_is_local(config, fused) -> bool:
kind = config.get("type") if isinstance(config, dict) else None
if kind in ("ByteLevel", "ByteFallback"):
# Both map whole pieces; after Fuse or ByteLevel joins them the piece is the whole reply.
return not fused
if kind in ("Metaspace", "Fuse"):
return True
if kind == "Replace":
pattern = config.get("pattern") or {}
return isinstance(pattern.get("String"), str) and len(pattern["String"]) == 1
if kind == "Strip":
return config.get("stop") == 0 and config.get("start") in (0, 1)
return False
def _decoder_is_local(config) -> bool:
steps = _decoder_steps(config)
if not steps:
return False
fused = False
for step in steps:
if not _step_is_local(step, fused):
return False
fused = fused or step.get("type") in ("Fuse", "ByteLevel")
return True
def _fast_tokenizer(tokenizer):
try:
from transformers import PreTrainedTokenizerFast
from transformers.tokenization_utils_base import PreTrainedTokenizerBase
except Exception: # noqa: BLE001 -- no transformers means no known decode path
return None
try:
bound = tokenizer.decode
except Exception: # noqa: BLE001 -- adapters vary
return None
inner = getattr(bound, "__self__", None)
if not isinstance(inner, PreTrainedTokenizerFast):
return None
cls = type(inner)
if (
getattr(bound, "__func__", None) is not PreTrainedTokenizerBase.decode
or cls._decode is not PreTrainedTokenizerFast._decode
or cls.clean_up_tokenization is not PreTrainedTokenizerBase.clean_up_tokenization
or any(
name in getattr(inner, "__dict__", {}) for name in ("_decode", "clean_up_tokenization")
)
):
return None
return inner
class _SplitProfile:
def __init__(self, tokenizer, added_ids, size):
# A strong ref would keep this _PROFILES key alive; IncrementalDecode holds the tokenizer.
self._tokenizer = weakref.proxy(tokenizer)
self._added = added_ids
self.size = size
self._whole = {}
self._ends_clean = {}
self._lock = threading.Lock()
def _whole_piece(self, token_id: int) -> bool:
whole = self._whole.get(token_id)
if whole is None:
whole, text, later = False, "", ""
if token_id not in self._added:
try:
piece = self._tokenizer.convert_ids_to_tokens(token_id)
if isinstance(piece, str) and not _BYTE_PIECE.fullmatch(piece):
text = self._tokenizer.decode([token_id], **_SPLIT_DECODE)
whole = isinstance(text, str) and bool(text) and "\ufffd" not in text
if whole:
# Metaspace strips "▁" from a run's first id only: "a▁" is "a" alone, "a " mid-reply.
pair = self._tokenizer.decode([token_id, token_id], **_SPLIT_DECODE)
later = pair[len(text) :] if pair.startswith(text) else ""
whole = bool(later) and "\ufffd" not in later
except Exception: # noqa: BLE001 -- an id the tokenizer cannot name never splits
whole = False
with self._lock:
self._whole[token_id] = whole
self._ends_clean[token_id] = (
whole and text[-1] not in _CLEANUP_CHARS and later[-1] not in _CLEANUP_CHARS
)
return whole
def splits(self, left: int, right: int, cleanup: bool) -> bool:
if not (self._whole_piece(left) and self._whole_piece(right)):
return False
return not cleanup or self._ends_clean[left]
_PROFILES = weakref.WeakKeyDictionary()
_PROFILES_LOCK = threading.Lock()
def split_profile(tokenizer):
"""The split profile for ``tokenizer``, or None when its decode is not known to split."""
inner = _fast_tokenizer(tokenizer)
if inner is None:
return None
try:
# Not len(inner): tokenizers < 0.23 rebuilds the whole vocab for it.
backend = inner.backend_tokenizer
size = (
backend.get_vocab_size(with_added_tokens = False),
len(backend.get_added_tokens_decoder()),
)
except Exception: # noqa: BLE001
return None
with _PROFILES_LOCK:
profile = _PROFILES.get(inner)
if profile is not None and profile.size == size:
return profile or None
try:
decoder = inner.backend_tokenizer.decoder
config = json.loads(decoder.__getstate__()) if decoder is not None else None
local = _decoder_is_local(config)
added = frozenset(int(token_id) for token_id in inner.added_tokens_decoder)
except Exception: # noqa: BLE001 -- anything unreadable keeps the full decode
local, added = False, frozenset()
profile = _SplitProfile(inner, added, size) if local else None
try:
with _PROFILES_LOCK:
_PROFILES[inner] = profile if profile is not None else _NoSplit(size)
except TypeError:
pass
return profile
class _NoSplit:
def __init__(self, size):
self.size = size
def __bool__(self):
return False
class IncrementalDecode:
"""``tokenizer.decode(ids, **decode_kwargs)`` of the ids fed so far. ``fallback(ids)`` replaces
that decode where the tokenizer does not split and must equal it where it does."""
def __init__(
self,
tokenizer,
decode_kwargs = None,
*,
fallback = None,
):
self._tokenizer = tokenizer
self._kwargs = dict(decode_kwargs or {})
self._fallback = fallback
self._profile = split_profile(tokenizer)
cleanup = self._kwargs.get("clean_up_tokenization_spaces")
if cleanup is None and self._profile is not None:
cleanup = getattr(self._profile._tokenizer, "clean_up_tokenization_spaces", True)
self._cleanup = cleanup is not False
self.reset()
def reset(self):
self.ids = []
self.text = ""
self._anchor = 0
self._anchor_text = ""
self._context_text = ""
def _decode(self, ids):
if self._fallback is not None:
return self._fallback(ids)
return self._tokenizer.decode(ids, **self._kwargs)
def feed(self, token_id) -> str:
ids = self.ids
ids.append(int(token_id))
previous = self.text
if self._profile is None or self._anchor == 0:
self.text = self._decode(ids)
else:
window = self._tokenizer.decode(ids[self._anchor - 1 :], **self._kwargs)
if window.startswith(self._context_text):
self.text = self._anchor_text + window[len(self._context_text) :]
else:
# Not reachable for a decoder that passed the gate; never guess.
self._profile = None
self.text = self._decode(ids)
count = len(ids)
if (
self._profile is not None
and count >= 2
and self._profile.splits(ids[-2], ids[-1], self._cleanup)
):
self._anchor = count - 1
self._anchor_text = previous
self._context_text = self._tokenizer.decode([ids[-2]], **self._kwargs)
return self.text
def __call__(self, ids) -> str:
count = len(ids)
if count == len(self.ids) + 1:
return self.feed(ids[-1])
if count != len(self.ids):
self.reset()
for token_id in ids:
self.feed(token_id)
return self.text