Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: hash the tokenizer fingerprint without touching the live tokenizer (#15749)

Daniel Hanb4e5559959f2
diff --git a/studio/backend/tests/test_tokenizer_fingerprint.py b/studio/backend/tests/test_tokenizer_fingerprint.py
index c7fa1973404..ba39bd2d730 100644
--- a/studio/backend/tests/test_tokenizer_fingerprint.py
+++ b/studio/backend/tests/test_tokenizer_fingerprint.py
@@ -144,7 +144,7 @@ def test_hashing_still_goes_through_datasets_own_reducer(restore_reducer):
tok = _tokenizer()
vocab, merges = tok._vocab, tok._merges
_hash(tok)
- assert seen == [[]]
+ assert seen == [["_merges", "_vocab"]]
assert tok._vocab is vocab and tok._merges is merges
@@ -155,7 +155,7 @@ def test_init_state_is_put_back_when_the_reducer_raises(restore_reducer):
tok = _tokenizer()
with pytest.raises(Exception):
_hash(tok)
- assert seen == [[]]
+ assert seen == [["_merges", "_vocab"]]
assert tok._vocab == {"<unk>": 0, "a": 1, "b": 2, "ab": 3}
assert tok._merges == [("a", "b")]
@@ -173,3 +173,46 @@ def test_backend_truncation_does_not_move_the_hash_where_datasets_ignores_it(res
tok(["ab ab ab ab"], truncation = True, max_length = 2)
assert tok._tokenizer.truncation is not None
assert _hash(tok) == before
+
+
+def test_serializing_the_tokenizer_during_a_hash_keeps_init_state(restore_reducer):
+ clones: list = []
+
+ def overlapping_serializer(pickler, obj):
+ clones.append(pickle.loads(pickle.dumps(obj)))
+ return restore_reducer(pickler, obj)
+
+ datasets_dill._save_transformersPreTrainedTokenizerBase = overlapping_serializer
+ datasets_dill.Pickler.dispatch.pop(transformers.PreTrainedTokenizerFast, None)
+ tokenizer_fingerprint.install_tokenizer_fingerprint()
+ tok = _tokenizer()
+ _hash(tok)
+ assert len(clones) == 1
+ assert clones[0].__dict__.get("_vocab") == tok._vocab
+ assert clones[0].__dict__.get("_merges") == tok._merges
+
+
+def test_fingerprint_equals_datasets_hash_without_init_state(restore_reducer):
+ tok = _tokenizer()
+ bare = _tokenizer()
+ del bare._vocab, bare._merges
+ expected = _hash(bare)
+ tokenizer_fingerprint.install_tokenizer_fingerprint()
+ assert _hash(tok) == expected
+
+
+class _VocabLookupTokenizer(transformers.PreTrainedTokenizerFast):
+ """A custom subclass that reads its Python vocab while encoding."""
+
+ def _convert_token_to_id_with_added_voc(self, token):
+ return self._vocab.get(token, 0)
+
+
+def test_custom_subclass_keeps_init_state_in_the_hash(restore_reducer):
+ tokenizer_fingerprint.install_tokenizer_fingerprint()
+ tok = _tokenizer()
+ tok.__class__ = _VocabLookupTokenizer
+ before = _hash(tok)
+ tok._vocab["ab"] = 7
+ assert tok.convert_tokens_to_ids("ab") == 7
+ assert _hash(tok) != before
diff --git a/studio/backend/utils/datasets/tokenizer_fingerprint.py b/studio/backend/utils/datasets/tokenizer_fingerprint.py
index 06dfcd9d13f..03a71effa2e 100644
--- a/studio/backend/utils/datasets/tokenizer_fingerprint.py
+++ b/studio/backend/utils/datasets/tokenizer_fingerprint.py
@@ -3,9 +3,9 @@
"""Hash tokenizers for datasets cache names without transformers 5's init-only ``_vocab`` / ``_merges``.
-They are read only in ``__init__`` to build the Rust ``_tokenizer``, which is hashed too and carries the
-same vocabulary, so dropping them keeps the fingerprint tied to vocab, added tokens and chat template.
-Only datasets' hashing Pickler is patched: multiprocess workers still receive the whole tokenizer.
+Read only in ``__init__`` to build the hashed Rust ``_tokenizer``; only datasets' hashing Pickler is patched,
+the live tokenizer is never modified, and only transformers' own classes qualify (custom or remote-code
+subclasses may read ``_vocab`` while encoding).
"""
_INIT_ONLY_STATE = ("_vocab", "_merges")
@@ -17,6 +17,35 @@ def _is_rust_tokenizer(value) -> bool:
return isinstance(value, Tokenizer)
+def _is_transformers_class(cls) -> bool:
+ # Remote code lives in ``transformers_modules``, which this prefix does not match.
+ module = getattr(cls, "__module__", None) or ""
+ return module == "transformers" or module.startswith("transformers.")
+
+
+def _without_init_only(state):
+ return {key: value for key, value in state.items() if key not in _INIT_ONLY_STATE}
+
+
+class _InitStateFilter:
+ """Pickler proxy dropping the init-only keys from one object's emitted state, so concurrent readers see it whole."""
+
+ def __init__(self, pickler, target):
+ self._pickler = pickler
+ self._target = target
+
+ def __getattr__(self, name):
+ return getattr(self._pickler, name)
+
+ def save_reduce(self, *args, **kwargs):
+ if kwargs.get("obj") is self._target:
+ if isinstance(kwargs.get("state"), dict):
+ kwargs["state"] = _without_init_only(kwargs["state"])
+ elif len(args) > 2 and isinstance(args[2], dict):
+ args = args[:2] + (_without_init_only(args[2]),) + args[3:]
+ return self._pickler.save_reduce(*args, **kwargs)
+
+
def install_tokenizer_fingerprint() -> bool:
"""Idempotent; False when this datasets has no tokenizer reducer to replace."""
try:
@@ -33,16 +62,14 @@ def install_tokenizer_fingerprint() -> bool:
def _save_tokenizer(pickler, obj):
state = obj.__dict__
- if not _is_rust_tokenizer(state.get("_tokenizer")) or not any(
- key in state for key in _INIT_ONLY_STATE
+ if (
+ not _is_transformers_class(type(obj))
+ or not _is_rust_tokenizer(state.get("_tokenizer"))
+ or not any(key in state for key in _INIT_ONLY_STATE)
):
return original(pickler, obj)
# Delegate so whatever datasets keeps out of the hash (encode cache, 4.7+ truncation / padding) stays out.
- held = {key: state.pop(key) for key in _INIT_ONLY_STATE if key in state}
- try:
- return original(pickler, obj)
- finally:
- state.update(held)
+ return original(_InitStateFilter(pickler, obj), obj)
setattr(_save_tokenizer, _MARKER, True)
_save_tokenizer.__wrapped__ = original