Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Run Unsloth's patched forwards only on Unsloth-loaded models (#15101)
Daniel Han8c69315827d5
diff --git a/tests/test_plain_transformers_after_fast_language_model.py b/tests/test_plain_transformers_after_fast_language_model.pynew file mode 100644index 00000000000..4fc07942116--- /dev/null+++ b/tests/test_plain_transformers_after_fast_language_model.py@@ -0,0 +1,146 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A plain transformers model loaded after FastLanguageModel in the same process runs the stock forwards."""++import inspect+import json+import subprocess+import sys+import types++import pytest+from real_accelerator import has_real_cuda+++def _fake_family(monkeypatch):+ modeling = types.ModuleType("transformers.models.fakeplain.modeling_fakeplain")+ patcher_module = types.ModuleType("unsloth.models.fakeplain")++ def hf_forward(+ self,+ input_ids = None,+ labels = None,+ ):+ return "hf"++ base = type(+ "FakeBase", (), {"__module__": modeling.__name__, "generate": lambda self: "hf gen"}+ )+ model_cls = type("FakeModel", (base,), {"__module__": modeling.__name__, "forward": hf_forward})+ modeling.FakeModel = model_cls+ patcher = type("FastFakeModel", (), {"__module__": patcher_module.__name__})+ patcher_module.FakeModel, patcher_module.FastFakeModel = model_cls, patcher+ monkeypatch.setitem(sys.modules, modeling.__name__, modeling)+ monkeypatch.setitem(sys.modules, patcher_module.__name__, patcher_module)+ return model_cls, patcher+++def test_patched_methods_only_run_on_marked_instances(monkeypatch):+ from unsloth.models import llama as unsloth_llama++ model_cls, patcher = _fake_family(monkeypatch)++ def fast_forward(+ self,+ input_ids = None,+ causal_mask = None,+ labels = None,+ ):+ return "fast"++ def pre_patch():+ model_cls.forward = fast_forward+ model_cls.generate = lambda self: "fast gen"+ model_cls.__init__ = lambda self: None++ for _ in range(2): # a second load re-patches; it must not wrap twice or lose the originals+ snapshot = unsloth_llama._snapshot_transformers_modules(patcher)+ pre_patch()+ unsloth_llama._record_pre_patch_changes(snapshot)+ unsloth_llama._gate_pre_patch_to_unsloth_models(snapshot)++ plain, marked = model_cls(), model_cls()+ marked._unsloth_fast_path = True+ assert plain.forward() == "hf" and plain.generate() == "hf gen"+ assert marked.forward() == "fast" and marked.generate() == "fast gen"+ assert vars(model_cls)["forward"].__wrapped__ is fast_forward+ # Trainer reads the forward signature to keep dataset columns.+ assert list(inspect.signature(model_cls.forward).parameters) == [+ "self",+ "input_ids",+ "causal_mask",+ "labels",+ ]+ assert vars(model_cls)["__init__"] is not None and not hasattr(+ vars(model_cls)["__init__"], "_unsloth_gated"+ )++ unsloth_llama.restore_transformers_family(["fakeplain"])+ assert marked.forward() == "hf" and marked.generate() == "hf gen"+ assert "generate" not in vars(model_cls)+++_TINY = """+import torch+from tokenizers import Tokenizer, models+from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedTokenizerFast+vocab = {{"<unk>": 0, "<s>": 1, "</s>": 2, **{{f"w{{i}}": i + 3 for i in range(125)}}}}+tok = PreTrainedTokenizerFast(tokenizer_object = Tokenizer(models.WordLevel(vocab, unk_token = "<unk>")),+ unk_token = "<unk>", bos_token = "<s>", eos_token = "</s>", pad_token = "</s>")+config = LlamaConfig(vocab_size = len(vocab), hidden_size = 64, intermediate_size = 128, num_hidden_layers = 2,+ num_attention_heads = 4, num_key_value_heads = 2, head_dim = 16, max_position_embeddings = 128,+ bos_token_id = 1, eos_token_id = 2, pad_token_id = 2)+path = {path!r}+torch.manual_seed(0)+LlamaForCausalLM(config).save_pretrained(path)+tok.save_pretrained(path)+"""++_PLAIN = """+import json, torch+from transformers import AutoModelForCausalLM+plain = AutoModelForCausalLM.from_pretrained({path!r}, dtype = torch.bfloat16).cuda().eval()+x = dict(input_ids = torch.tensor([[1, 5, 6, 7, 8, 9]], device = "cuda"),+ attention_mask = torch.ones(1, 6, dtype = torch.long, device = "cuda"))+with torch.no_grad():+ logits = plain(**x).logits.float()+gen = plain.generate(**x, max_new_tokens = 8, do_sample = False, min_new_tokens = 8)+print("PLAIN", json.dumps({{"logits": logits.cpu().tolist(), "gen": gen.cpu().tolist()}}))+"""+++def _run(code):+ out = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, timeout = 900)+ lines = [line for line in out.stdout.splitlines() if line.startswith(("PLAIN ", "SECOND "))]+ assert out.returncode == 0 and lines, (out.stdout[-2000:], out.stderr[-3000:])+ return {line.split(" ", 1)[0]: json.loads(line.split(" ", 1)[1]) for line in lines}++[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need CUDA")+def test_plain_transformers_after_fast_language_model_matches_a_fresh_process(tmp_path):+ import torch++ path = str(tmp_path / "tiny")+ fresh = _run(_TINY.format(path = path) + _PLAIN.format(path = path))["PLAIN"]+ after = _run(+ f"""+import os+os.environ["UNSLOTH_COMPILE_LOCATION"] = {str(tmp_path / "compiled")!r}+from unsloth import FastLanguageModel+import json, torch+model, _ = FastLanguageModel.from_pretrained({path!r}, load_in_4bit = False, max_seq_length = 64)+"""+ + _PLAIN.format(path = path)+ + f"""+again, _ = FastLanguageModel.from_pretrained({path!r}, load_in_4bit = False, max_seq_length = 64)+with torch.no_grad():+ a = model(input_ids = x["input_ids"]).logits.float()+ b = again(input_ids = x["input_ids"]).logits.float()+print("SECOND", json.dumps({{"diff": (a - b).abs().max().item()}}))+"""+ )+ assert after["PLAIN"]["gen"] == fresh["gen"]+ diff = (torch.tensor(after["PLAIN"]["logits"]) - torch.tensor(fresh["logits"])).abs().max()+ assert diff.item() < 1e-2, diff # bf16; the RoPE tables are built on a different device+ assert after["SECOND"]["diff"] == 0.0diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex db388bf568e..1cc5261a7c7 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2374,6 +2374,16 @@ class _PerDeviceRopeCache:position_ids = None,seq_len = None,):+ if position_ids is not None and seq_len is None:+ # Stock transformers call (a plain model built after the patch): per-position cos/sin.+ rows = int(position_ids.max()) + 1+ if rows > self.current_rope_size:+ self._grow_rope_cache(rows, x.dtype)+ self._need_rope_rows(rows)+ device_index = self._ensure_rope_cache(x.device.index)+ cos, sin = self._slice_rope_cache(device_index, rows)+ position_ids = position_ids.to(cos.device)+ return cos[position_ids].to(x.dtype), sin[position_ids].to(x.dtype)if seq_len is not None and seq_len > self.current_rope_size:self._grow_rope_cache(seq_len, x.dtype)self._need_rope_rows(self.current_rope_size if seq_len is None else seq_len)@@ -3094,6 +3104,43 @@ def _record_pre_patch_changes(snapshot):changed_attrs.setdefault((cls, name), saved.get(name, _MISSING))+def _gate_pre_patch_to_unsloth_models(snapshot):+ """pre_patch edits transformers classes; run patched methods only on Unsloth-marked modules, else+ the pre-patch original (a plain model built later lacks the state the fast paths need)."""+ for modeling in snapshot:+ record = vars(modeling).get("_unsloth_pre_patch_originals")+ if record is None:+ continue+ for (cls, name), original in record[1].items():+ fast = vars(cls).get(name, _MISSING)+ if name.startswith("__") or not inspect.isfunction(fast):+ continue+ if getattr(fast, "_unsloth_gated", False):+ continue+ if original is not _MISSING and not callable(original):+ continue+ setattr(cls, name, _gated_method(cls, name, fast, original))+++def _gated_method(cls, name, fast, original):+ @functools.wraps(fast)+ def gated(self, *args, **kwargs):+ if getattr(self, "_unsloth_fast_path", False):+ return fast(self, *args, **kwargs)+ if original is _MISSING:+ return getattr(super(cls, self), name)(*args, **kwargs)+ return original(self, *args, **kwargs)++ gated._unsloth_gated = True+ return gated+++def mark_unsloth_fast_path(model):+ """Opt every module of an Unsloth-loaded model into the patched forwards."""+ for module in model.modules():+ module._unsloth_fast_path = True++def restore_transformers_family(model_types):"""Undo pre_patch before FastModel compiles a family: the compiler copies whatever forwards the classes hold."""for model_type in model_types:@@ -3373,6 +3420,7 @@ class FastLlamaModel:snapshot = _snapshot_transformers_modules(model_patcher)model_patcher.pre_patch()_record_pre_patch_changes(snapshot)+ _gate_pre_patch_to_unsloth_models(snapshot)# A download counter, to see whether environments are breaking or HF is down.get_statistics(kwargs.get("local_files_only", False))@@ -4121,6 +4169,7 @@ class FastLlamaModel:**_tokenizer_cache_kwargs,)+ mark_unsloth_fast_path(model)model, tokenizer = patch_tokenizer(model, tokenizer)model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)set_deepspeed_moe_leaf_modules(model)@@ -5259,6 +5308,7 @@ class FastLlamaModel:# Since transformers 4.53, must turn off explicitlyfor module in model.modules():module.max_seq_length = max_seq_length+ mark_unsloth_fast_path(model)internal_model = modelwhile hasattr(internal_model, "model"):