Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Run Unsloth's patched forwards only on Unsloth-loaded models (#15101)

Daniel Han8c69315827d5
diff --git a/tests/test_plain_transformers_after_fast_language_model.py b/tests/test_plain_transformers_after_fast_language_model.py
new file mode 100644
index 00000000000..4fc07942116
--- /dev/null
+++ b/tests/test_plain_transformers_after_fast_language_model.py
@@ -0,0 +1,146 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A plain transformers model loaded after FastLanguageModel in the same process runs the stock forwards."""
+
+import inspect
+import json
+import subprocess
+import sys
+import types
+
+import pytest
+from real_accelerator import has_real_cuda
+
+
+def _fake_family(monkeypatch):
+ modeling = types.ModuleType("transformers.models.fakeplain.modeling_fakeplain")
+ patcher_module = types.ModuleType("unsloth.models.fakeplain")
+
+ def hf_forward(
+ self,
+ input_ids = None,
+ labels = None,
+ ):
+ return "hf"
+
+ base = type(
+ "FakeBase", (), {"__module__": modeling.__name__, "generate": lambda self: "hf gen"}
+ )
+ model_cls = type("FakeModel", (base,), {"__module__": modeling.__name__, "forward": hf_forward})
+ modeling.FakeModel = model_cls
+ patcher = type("FastFakeModel", (), {"__module__": patcher_module.__name__})
+ patcher_module.FakeModel, patcher_module.FastFakeModel = model_cls, patcher
+ monkeypatch.setitem(sys.modules, modeling.__name__, modeling)
+ monkeypatch.setitem(sys.modules, patcher_module.__name__, patcher_module)
+ return model_cls, patcher
+
+
+def test_patched_methods_only_run_on_marked_instances(monkeypatch):
+ from unsloth.models import llama as unsloth_llama
+
+ model_cls, patcher = _fake_family(monkeypatch)
+
+ def fast_forward(
+ self,
+ input_ids = None,
+ causal_mask = None,
+ labels = None,
+ ):
+ return "fast"
+
+ def pre_patch():
+ model_cls.forward = fast_forward
+ model_cls.generate = lambda self: "fast gen"
+ model_cls.__init__ = lambda self: None
+
+ for _ in range(2): # a second load re-patches; it must not wrap twice or lose the originals
+ snapshot = unsloth_llama._snapshot_transformers_modules(patcher)
+ pre_patch()
+ unsloth_llama._record_pre_patch_changes(snapshot)
+ unsloth_llama._gate_pre_patch_to_unsloth_models(snapshot)
+
+ plain, marked = model_cls(), model_cls()
+ marked._unsloth_fast_path = True
+ assert plain.forward() == "hf" and plain.generate() == "hf gen"
+ assert marked.forward() == "fast" and marked.generate() == "fast gen"
+ assert vars(model_cls)["forward"].__wrapped__ is fast_forward
+ # Trainer reads the forward signature to keep dataset columns.
+ assert list(inspect.signature(model_cls.forward).parameters) == [
+ "self",
+ "input_ids",
+ "causal_mask",
+ "labels",
+ ]
+ assert vars(model_cls)["__init__"] is not None and not hasattr(
+ vars(model_cls)["__init__"], "_unsloth_gated"
+ )
+
+ unsloth_llama.restore_transformers_family(["fakeplain"])
+ assert marked.forward() == "hf" and marked.generate() == "hf gen"
+ assert "generate" not in vars(model_cls)
+
+
+_TINY = """
+import torch
+from tokenizers import Tokenizer, models
+from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedTokenizerFast
+vocab = {{"<unk>": 0, "<s>": 1, "</s>": 2, **{{f"w{{i}}": i + 3 for i in range(125)}}}}
+tok = PreTrainedTokenizerFast(tokenizer_object = Tokenizer(models.WordLevel(vocab, unk_token = "<unk>")),
+ unk_token = "<unk>", bos_token = "<s>", eos_token = "</s>", pad_token = "</s>")
+config = LlamaConfig(vocab_size = len(vocab), hidden_size = 64, intermediate_size = 128, num_hidden_layers = 2,
+ num_attention_heads = 4, num_key_value_heads = 2, head_dim = 16, max_position_embeddings = 128,
+ bos_token_id = 1, eos_token_id = 2, pad_token_id = 2)
+path = {path!r}
+torch.manual_seed(0)
+LlamaForCausalLM(config).save_pretrained(path)
+tok.save_pretrained(path)
+"""
+
+_PLAIN = """
+import json, torch
+from transformers import AutoModelForCausalLM
+plain = AutoModelForCausalLM.from_pretrained({path!r}, dtype = torch.bfloat16).cuda().eval()
+x = dict(input_ids = torch.tensor([[1, 5, 6, 7, 8, 9]], device = "cuda"),
+ attention_mask = torch.ones(1, 6, dtype = torch.long, device = "cuda"))
+with torch.no_grad():
+ logits = plain(**x).logits.float()
+gen = plain.generate(**x, max_new_tokens = 8, do_sample = False, min_new_tokens = 8)
+print("PLAIN", json.dumps({{"logits": logits.cpu().tolist(), "gen": gen.cpu().tolist()}}))
+"""
+
+
+def _run(code):
+ out = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, timeout = 900)
+ lines = [line for line in out.stdout.splitlines() if line.startswith(("PLAIN ", "SECOND "))]
+ assert out.returncode == 0 and lines, (out.stdout[-2000:], out.stderr[-3000:])
+ return {line.split(" ", 1)[0]: json.loads(line.split(" ", 1)[1]) for line in lines}
+
+
[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need CUDA")
+def test_plain_transformers_after_fast_language_model_matches_a_fresh_process(tmp_path):
+ import torch
+
+ path = str(tmp_path / "tiny")
+ fresh = _run(_TINY.format(path = path) + _PLAIN.format(path = path))["PLAIN"]
+ after = _run(
+ f"""
+import os
+os.environ["UNSLOTH_COMPILE_LOCATION"] = {str(tmp_path / "compiled")!r}
+from unsloth import FastLanguageModel
+import json, torch
+model, _ = FastLanguageModel.from_pretrained({path!r}, load_in_4bit = False, max_seq_length = 64)
+"""
+ + _PLAIN.format(path = path)
+ + f"""
+again, _ = FastLanguageModel.from_pretrained({path!r}, load_in_4bit = False, max_seq_length = 64)
+with torch.no_grad():
+ a = model(input_ids = x["input_ids"]).logits.float()
+ b = again(input_ids = x["input_ids"]).logits.float()
+print("SECOND", json.dumps({{"diff": (a - b).abs().max().item()}}))
+"""
+ )
+ assert after["PLAIN"]["gen"] == fresh["gen"]
+ diff = (torch.tensor(after["PLAIN"]["logits"]) - torch.tensor(fresh["logits"])).abs().max()
+ assert diff.item() < 1e-2, diff # bf16; the RoPE tables are built on a different device
+ assert after["SECOND"]["diff"] == 0.0
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index db388bf568e..1cc5261a7c7 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2374,6 +2374,16 @@ class _PerDeviceRopeCache:
position_ids = None,
seq_len = None,
):
+ if position_ids is not None and seq_len is None:
+ # Stock transformers call (a plain model built after the patch): per-position cos/sin.
+ rows = int(position_ids.max()) + 1
+ if rows > self.current_rope_size:
+ self._grow_rope_cache(rows, x.dtype)
+ self._need_rope_rows(rows)
+ device_index = self._ensure_rope_cache(x.device.index)
+ cos, sin = self._slice_rope_cache(device_index, rows)
+ position_ids = position_ids.to(cos.device)
+ return cos[position_ids].to(x.dtype), sin[position_ids].to(x.dtype)
if seq_len is not None and seq_len > self.current_rope_size:
self._grow_rope_cache(seq_len, x.dtype)
self._need_rope_rows(self.current_rope_size if seq_len is None else seq_len)
@@ -3094,6 +3104,43 @@ def _record_pre_patch_changes(snapshot):
changed_attrs.setdefault((cls, name), saved.get(name, _MISSING))
+def _gate_pre_patch_to_unsloth_models(snapshot):
+ """pre_patch edits transformers classes; run patched methods only on Unsloth-marked modules, else
+ the pre-patch original (a plain model built later lacks the state the fast paths need)."""
+ for modeling in snapshot:
+ record = vars(modeling).get("_unsloth_pre_patch_originals")
+ if record is None:
+ continue
+ for (cls, name), original in record[1].items():
+ fast = vars(cls).get(name, _MISSING)
+ if name.startswith("__") or not inspect.isfunction(fast):
+ continue
+ if getattr(fast, "_unsloth_gated", False):
+ continue
+ if original is not _MISSING and not callable(original):
+ continue
+ setattr(cls, name, _gated_method(cls, name, fast, original))
+
+
+def _gated_method(cls, name, fast, original):
+ @functools.wraps(fast)
+ def gated(self, *args, **kwargs):
+ if getattr(self, "_unsloth_fast_path", False):
+ return fast(self, *args, **kwargs)
+ if original is _MISSING:
+ return getattr(super(cls, self), name)(*args, **kwargs)
+ return original(self, *args, **kwargs)
+
+ gated._unsloth_gated = True
+ return gated
+
+
+def mark_unsloth_fast_path(model):
+ """Opt every module of an Unsloth-loaded model into the patched forwards."""
+ for module in model.modules():
+ module._unsloth_fast_path = True
+
+
def restore_transformers_family(model_types):
"""Undo pre_patch before FastModel compiles a family: the compiler copies whatever forwards the classes hold."""
for model_type in model_types:
@@ -3373,6 +3420,7 @@ class FastLlamaModel:
snapshot = _snapshot_transformers_modules(model_patcher)
model_patcher.pre_patch()
_record_pre_patch_changes(snapshot)
+ _gate_pre_patch_to_unsloth_models(snapshot)
# A download counter, to see whether environments are breaking or HF is down.
get_statistics(kwargs.get("local_files_only", False))
@@ -4121,6 +4169,7 @@ class FastLlamaModel:
**_tokenizer_cache_kwargs,
)
+ mark_unsloth_fast_path(model)
model, tokenizer = patch_tokenizer(model, tokenizer)
model, tokenizer = model_patcher.post_patch(model, tokenizer, correct_dtype = dtype)
set_deepspeed_moe_leaf_modules(model)
@@ -5259,6 +5308,7 @@ class FastLlamaModel:
# Since transformers 4.53, must turn off explicitly
for module in model.modules():
module.max_seq_length = max_seq_length
+ mark_unsloth_fast_path(model)
internal_model = model
while hasattr(internal_model, "model"):