Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: retry from the Hub when a cached snapshot has no tokenizer files (#13412)
Daniel Hanf2e1e107893a
diff --git a/studio/backend/core/training/trainer.py b/studio/backend/core/training/trainer.pyindex 45ee91c8bd0..62405a6ed8b 100644--- a/studio/backend/core/training/trainer.py+++ b/studio/backend/core/training/trainer.py@@ -402,6 +402,15 @@ def _ignored_adapter_settings(return ignored+def _tokenizer_is_empty(tokenizer) -> bool:+ # Not a length check: a snapshot keeping tokenizer_config.json but no vocab still lists its special tokens.+ inner = getattr(tokenizer, "tokenizer", tokenizer)+ try:+ return not inner("hello world", add_special_tokens = False)["input_ids"]+ except Exception:+ return False++def _load_adapter_aware_tokenizer(lookup_name, hf_token, **kwargs):"""AutoTokenizer for *lookup_name*. A PEFT adapter saved without tokenizer files falls backto the base it names, as Unsloth's own adapter loader does (#13140)."""@@ -566,6 +575,17 @@ class UnslothTrainer:# missing sentencepiece that is installed and irrelevant.**_spark_tts_tokenizer_kwargs(self._audio_type, lookup_name),)+ # Transformers 5 builds a one-token tokenizer from a snapshot without vocab files instead of raising, which+ # skipped the incomplete-cache Hub retry and trained to NaN loss (#6958).+ if local_files_only and _tokenizer_is_empty(self.tokenizer):+ raise FileNotFoundError(+ f"The cached snapshot of {model_name} has no tokenizer files."+ )+ if model_revision and _tokenizer_is_empty(self.tokenizer):+ raise FileNotFoundError(+ f"{model_name} has no tokenizer files at revision {model_revision}, the revision of "+ "its cached copy. Delete the cached model and retry to use the latest revision."+ )# The probe and the tokenizer load are separate reads of the same repo, so a transient 5xx can leave the probe# inconclusive while the read after it succeeds; detect_audio_type_checked caches only definitive results.diff --git a/studio/backend/tests/test_training_tokenizerless_snapshot.py b/studio/backend/tests/test_training_tokenizerless_snapshot.pynew file mode 100644index 00000000000..5d0a5ac8b12--- /dev/null+++ b/studio/backend/tests/test_training_tokenizerless_snapshot.py@@ -0,0 +1,165 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A cached snapshot without tokenizer files must take the incomplete-cache Hub retry (#6958).++Transformers 5 loads such a snapshot as a one-token tokenizer instead of raising, so the retry+never ran and the run trained to NaN loss and reported success.+"""++from __future__ import annotations++import importlib+import json+import sys+import types+from unittest.mock import MagicMock++import pytest+++_STUBBED: list[str] = []+++def _stub_if_missing(name, attrs):+ """Same reason as test_audio_type_inconclusive.py: the backend pytest job has no unsloth."""+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001+ pass+ _STUBBED.append(name)+ mod = types.ModuleType(name)+ mod.__spec__ = None+ for attr in attrs:+ setattr(mod, attr, MagicMock())+ sys.modules[name] = mod+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, mod)+++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++import core.training.trainer as trainer_mod # noqa: E402+from core.training import worker # noqa: E402+from core.training.trainer import UnslothTrainer # noqa: E402++for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)++import transformers # noqa: E402++transformers.AutoProcessor # noqa: B018+transformers.AutoTokenizer # noqa: B018+transformers = sys.modules["transformers"]++_QWEN2_CONFIG = {+ "architectures": ["Qwen2ForCausalLM"],+ "model_type": "qwen2",+ "hidden_size": 64,+ "intermediate_size": 128,+ "num_attention_heads": 4,+ "num_hidden_layers": 2,+ "num_key_value_heads": 2,+ "vocab_size": 151936,+}+++def text_model(monkeypatch):+ monkeypatch.setattr(trainer_mod, "detect_audio_type_checked", lambda *a, **kw: (None, True))+ monkeypatch.setattr(trainer_mod, "is_vision_model", lambda *a, **kw: False)+++def _fake_tokenizer(monkeypatch, size):+ class _Tok:+ def __len__(self):+ return size++ def __call__(self, text, **kwargs):+ return {"input_ids": list(range(2)) if size > 1 else []}++ class _Auto:+ @classmethod+ def from_pretrained(cls, *a, **kw):+ return _Tok()++ monkeypatch.setattr(transformers, "AutoTokenizer", _Auto, raising = False)+++def test_tokenizerless_snapshot_is_a_cache_artifact_error(tmp_path, monkeypatch, text_model):+ snapshot = tmp_path / "snapshots" / "6834895"+ snapshot.mkdir(parents = True)+ (snapshot / "config.json").write_text(json.dumps(_QWEN2_CONFIG))+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")++ with pytest.raises(Exception) as caught:+ UnslothTrainer().pre_detect_and_load_tokenizer(+ "org/model", model_load_name = str(snapshot), local_files_only = True+ )++ # Transformers 4 raises its own TypeError here; either way the worker must retry from the Hub.+ assert worker._is_model_cache_artifact_error(caught.value)+++def test_snapshot_with_only_tokenizer_config_is_a_cache_artifact_error(+ tmp_path, monkeypatch, text_model+):+ # An interrupted tokenizer download keeps tokenizer_config.json and its special tokens, not the vocab.+ snapshot = tmp_path / "snapshots" / "6834895"+ snapshot.mkdir(parents = True)+ (snapshot / "config.json").write_text(json.dumps(_QWEN2_CONFIG))+ special = ["<|endoftext|>", "<|im_start|>", "<|im_end|>"]+ (snapshot / "tokenizer_config.json").write_text(+ json.dumps(+ {+ "tokenizer_class": "Qwen2Tokenizer",+ "eos_token": "<|im_end|>",+ "added_tokens_decoder": {+ str(151643 + i): {"content": t, "special": True} for i, t in enumerate(special)+ },+ }+ )+ )+ monkeypatch.setenv("HF_HUB_OFFLINE", "1")++ with pytest.raises(Exception) as caught:+ UnslothTrainer().pre_detect_and_load_tokenizer(+ "org/model", model_load_name = str(snapshot), local_files_only = True+ )++ assert worker._is_model_cache_artifact_error(caught.value)+++def test_pinned_revision_without_tokenizer_names_the_cause(monkeypatch, text_model):+ _fake_tokenizer(monkeypatch, 1)++ with pytest.raises(FileNotFoundError, match = "no tokenizer files at revision 6834895"):+ UnslothTrainer().pre_detect_and_load_tokenizer("org/model", model_revision = "6834895")+++ "kwargs",+ [{"local_files_only": True, "model_load_name": "/cache/snap"}, {"model_revision": "abc"}, {}],+)+def test_a_real_tokenizer_still_loads(monkeypatch, text_model, kwargs):+ _fake_tokenizer(monkeypatch, 151665)+ trainer = UnslothTrainer()++ trainer.pre_detect_and_load_tokenizer("org/model", **kwargs)++ assert len(trainer.tokenizer) == 151665+++def test_an_unpinned_hub_load_is_left_alone(monkeypatch, text_model):+ _fake_tokenizer(monkeypatch, 1)+ trainer = UnslothTrainer()++ trainer.pre_detect_and_load_tokenizer("org/model")++ assert len(trainer.tokenizer) == 1