Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: hash the tokenizer fingerprint without touching the live tokenizer (#15749)
Daniel Hanb4e5559959f2
diff --git a/studio/backend/tests/test_tokenizer_fingerprint.py b/studio/backend/tests/test_tokenizer_fingerprint.pyindex c7fa1973404..ba39bd2d730 100644--- a/studio/backend/tests/test_tokenizer_fingerprint.py+++ b/studio/backend/tests/test_tokenizer_fingerprint.py@@ -144,7 +144,7 @@ def test_hashing_still_goes_through_datasets_own_reducer(restore_reducer):tok = _tokenizer()vocab, merges = tok._vocab, tok._merges_hash(tok)- assert seen == [[]]+ assert seen == [["_merges", "_vocab"]]assert tok._vocab is vocab and tok._merges is merges@@ -155,7 +155,7 @@ def test_init_state_is_put_back_when_the_reducer_raises(restore_reducer):tok = _tokenizer()with pytest.raises(Exception):_hash(tok)- assert seen == [[]]+ assert seen == [["_merges", "_vocab"]]assert tok._vocab == {"<unk>": 0, "a": 1, "b": 2, "ab": 3}assert tok._merges == [("a", "b")]@@ -173,3 +173,46 @@ def test_backend_truncation_does_not_move_the_hash_where_datasets_ignores_it(restok(["ab ab ab ab"], truncation = True, max_length = 2)assert tok._tokenizer.truncation is not Noneassert _hash(tok) == before+++def test_serializing_the_tokenizer_during_a_hash_keeps_init_state(restore_reducer):+ clones: list = []++ def overlapping_serializer(pickler, obj):+ clones.append(pickle.loads(pickle.dumps(obj)))+ return restore_reducer(pickler, obj)++ datasets_dill._save_transformersPreTrainedTokenizerBase = overlapping_serializer+ datasets_dill.Pickler.dispatch.pop(transformers.PreTrainedTokenizerFast, None)+ tokenizer_fingerprint.install_tokenizer_fingerprint()+ tok = _tokenizer()+ _hash(tok)+ assert len(clones) == 1+ assert clones[0].__dict__.get("_vocab") == tok._vocab+ assert clones[0].__dict__.get("_merges") == tok._merges+++def test_fingerprint_equals_datasets_hash_without_init_state(restore_reducer):+ tok = _tokenizer()+ bare = _tokenizer()+ del bare._vocab, bare._merges+ expected = _hash(bare)+ tokenizer_fingerprint.install_tokenizer_fingerprint()+ assert _hash(tok) == expected+++class _VocabLookupTokenizer(transformers.PreTrainedTokenizerFast):+ """A custom subclass that reads its Python vocab while encoding."""++ def _convert_token_to_id_with_added_voc(self, token):+ return self._vocab.get(token, 0)+++def test_custom_subclass_keeps_init_state_in_the_hash(restore_reducer):+ tokenizer_fingerprint.install_tokenizer_fingerprint()+ tok = _tokenizer()+ tok.__class__ = _VocabLookupTokenizer+ before = _hash(tok)+ tok._vocab["ab"] = 7+ assert tok.convert_tokens_to_ids("ab") == 7+ assert _hash(tok) != beforediff --git a/studio/backend/utils/datasets/tokenizer_fingerprint.py b/studio/backend/utils/datasets/tokenizer_fingerprint.pyindex 06dfcd9d13f..03a71effa2e 100644--- a/studio/backend/utils/datasets/tokenizer_fingerprint.py+++ b/studio/backend/utils/datasets/tokenizer_fingerprint.py@@ -3,9 +3,9 @@"""Hash tokenizers for datasets cache names without transformers 5's init-only ``_vocab`` / ``_merges``.-They are read only in ``__init__`` to build the Rust ``_tokenizer``, which is hashed too and carries the-same vocabulary, so dropping them keeps the fingerprint tied to vocab, added tokens and chat template.-Only datasets' hashing Pickler is patched: multiprocess workers still receive the whole tokenizer.+Read only in ``__init__`` to build the hashed Rust ``_tokenizer``; only datasets' hashing Pickler is patched,+the live tokenizer is never modified, and only transformers' own classes qualify (custom or remote-code+subclasses may read ``_vocab`` while encoding)."""_INIT_ONLY_STATE = ("_vocab", "_merges")@@ -17,6 +17,35 @@ def _is_rust_tokenizer(value) -> bool:return isinstance(value, Tokenizer)+def _is_transformers_class(cls) -> bool:+ # Remote code lives in ``transformers_modules``, which this prefix does not match.+ module = getattr(cls, "__module__", None) or ""+ return module == "transformers" or module.startswith("transformers.")+++def _without_init_only(state):+ return {key: value for key, value in state.items() if key not in _INIT_ONLY_STATE}+++class _InitStateFilter:+ """Pickler proxy dropping the init-only keys from one object's emitted state, so concurrent readers see it whole."""++ def __init__(self, pickler, target):+ self._pickler = pickler+ self._target = target++ def __getattr__(self, name):+ return getattr(self._pickler, name)++ def save_reduce(self, *args, **kwargs):+ if kwargs.get("obj") is self._target:+ if isinstance(kwargs.get("state"), dict):+ kwargs["state"] = _without_init_only(kwargs["state"])+ elif len(args) > 2 and isinstance(args[2], dict):+ args = args[:2] + (_without_init_only(args[2]),) + args[3:]+ return self._pickler.save_reduce(*args, **kwargs)++def install_tokenizer_fingerprint() -> bool:"""Idempotent; False when this datasets has no tokenizer reducer to replace."""try:@@ -33,16 +62,14 @@ def install_tokenizer_fingerprint() -> bool:def _save_tokenizer(pickler, obj):state = obj.__dict__- if not _is_rust_tokenizer(state.get("_tokenizer")) or not any(- key in state for key in _INIT_ONLY_STATE+ if (+ not _is_transformers_class(type(obj))+ or not _is_rust_tokenizer(state.get("_tokenizer"))+ or not any(key in state for key in _INIT_ONLY_STATE)):return original(pickler, obj)# Delegate so whatever datasets keeps out of the hash (encode cache, 4.7+ truncation / padding) stays out.- held = {key: state.pop(key) for key in _INIT_ONLY_STATE if key in state}- try:- return original(pickler, obj)- finally:- state.update(held)+ return original(_InitStateFilter(pickler, obj), obj)setattr(_save_tokenizer, _MARKER, True)_save_tokenizer.__wrapped__ = original