Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Honour YaRN passed as rope_parameters, and rope_scaling on FastModel (#15214)
Daniel Han801ba6d049f0
diff --git a/tests/test_optimized_precision_conflicts.py b/tests/test_optimized_precision_conflicts.pyindex 034f1355191..5d8908b16f2 100644--- a/tests/test_optimized_precision_conflicts.py+++ b/tests/test_optimized_precision_conflicts.py@@ -59,7 +59,8 @@ def loader():nfor n in tree.bodyif isinstance(n, ast.FunctionDef)- and n.name in ("_precision_flags_conflict", "_modelscope_snapshot_or_none")+ and n.name+ in ("_precision_flags_conflict", "_modelscope_snapshot_or_none", "_fold_rope_parameters")]captured = {"config_calls": 0}diff --git a/tests/test_yarn_rope_parameters_load.py b/tests/test_yarn_rope_parameters_load.pynew file mode 100644index 00000000000..6c63dad3767--- /dev/null+++ b/tests/test_yarn_rope_parameters_load.py@@ -0,0 +1,217 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""YaRN passed to from_pretrained must reach the model, as `rope_scaling` or as transformers v5 /+TRL's `rope_parameters` spelling (SFTConfig(model_init_kwargs = {"rope_parameters": ...})).++Before: FastLanguageModel ignored `rope_parameters`, FastModel passed it to the model constructor+(TypeError), and FastModel dropped `rope_scaling` entirely, so the model trained on plain RoPE."""++import subprocess+import sys+from pathlib import Path++import pytest+from packaging.version import Version++transformers = pytest.importorskip("transformers")+pytest.importorskip("torch")++_TESTS = str(Path(__file__).resolve().parent)++# Subprocess: the loaders patch transformers process-wide, and the CPU redirects must not leak.+_CHECK = r"""+import math, os, sys+os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")+os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")+sys.path.insert(0, sys.argv[1])+import _zoo_aggressive_cuda_spoof as _spoof+_spoof.apply()+import torch+import unsloth+_is_cuda = lambda d: isinstance(d, (str, int, torch.device)) and not isinstance(d, bool) and torch.device(d).type == "cuda"+for _name in ("empty", "zeros", "ones", "full", "tensor", "arange", "randn", "rand", "randint",+ "empty_like", "zeros_like", "ones_like"):+ _orig = getattr(torch, _name)+ def _redir(*a, _orig = _orig, **k):+ if _is_cuda(k.get("device")):+ k["device"] = "cpu"+ return _orig(*a, **k)+ setattr(torch, _name, _redir)+_orig_to = torch.Tensor.to+def _to(self, *a, **k):+ a = tuple("cpu" if _is_cuda(x) else x for x in a)+ if _is_cuda(k.get("device")):+ k["device"] = "cpu"+ return _orig_to(self, *a, **k)+torch.Tensor.to = _to+torch.Tensor.cuda = lambda self, *a, **k: self+if hasattr(torch, "accelerator"):+ torch.accelerator.is_available = lambda *a, **k: False+from unsloth import FastLanguageModel, FastModel++loader, kwarg = sys.argv[2], sys.argv[3]+if loader == "unit":+ from unsloth.models.loader import _fold_rope_parameters+ from unsloth.models.vision import _apply_user_rope_scaling++ yarn = {"rope_type": "yarn", "factor": 4.0}+ kwargs = {"rope_parameters": dict(yarn), "other": 1}+ assert _fold_rope_parameters(None, kwargs) == yarn+ assert kwargs == {"other": 1}+ assert _fold_rope_parameters(dict(yarn), {"rope_parameters": dict(yarn)}) == yarn+ assert _fold_rope_parameters(yarn, {}) is yarn+ try:+ _fold_rope_parameters({"rope_type": "linear", "factor": 2.0}, {"rope_parameters": yarn})+ raise AssertionError("differing rope_scaling / rope_parameters accepted")+ except ValueError:+ pass++ class Cfg:+ rope_parameters = {+ "full_attention": {"rope_type": "default", "rope_theta": 1e6},+ "sliding_attention": {"rope_type": "default", "rope_theta": 1e4},+ }++ # A flat override applies to full attention only, as transformers' Gemma 3 config does.+ cfg = Cfg()+ assert _apply_user_rope_scaling(cfg, {"rope_type": "yarn", "factor": 4.0}) is None+ assert cfg.rope_parameters == {+ "full_attention": {"rope_type": "yarn", "factor": 4.0, "rope_theta": 1e6},+ "sliding_attention": {"rope_type": "default", "rope_theta": 1e4},+ }, cfg.rope_parameters++ # A flat config keeps its layout keys (Qwen2-VL mrope_section, StableLM partial_rotary_factor).+ class Flat:+ max_position_embeddings = 4096+ rope_parameters = {+ "rope_type": "default", "rope_theta": 1e4, "partial_rotary_factor": 0.25,+ "mrope_section": [2, 1, 1],+ }++ flat = Flat()+ applied = _apply_user_rope_scaling(flat, {"rope_type": "yarn", "factor": 4.0})+ expected = {+ "rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 4096,+ "rope_theta": 1e4, "partial_rotary_factor": 0.25, "mrope_section": [2, 1, 1],+ }+ assert applied == expected and flat.rope_scaling == expected, (applied, flat.rope_scaling)+ try:+ _apply_user_rope_scaling(Flat(), {"full_attention": {"rope_type": "yarn", "factor": 4.0}})+ raise AssertionError("per-layer-type override accepted for a flat config")+ except ValueError:+ pass+ new = {+ "full_attention": {"rope_type": "yarn", "factor": 4.0, "rope_theta": 1e6},+ "sliding_attention": {"rope_type": "default", "rope_theta": 1e4},+ }+ _apply_user_rope_scaling(cfg, new)+ assert cfg.rope_parameters == new+ # Entries without rope_theta keep the layer type's base (Gemma 3's rotary init indexes it).+ cfg = Cfg()+ _apply_user_rope_scaling(+ cfg,+ {+ "full_attention": {"rope_type": "yarn", "factor": 4.0},+ "sliding_attention": {"rope_type": "default"},+ },+ )+ assert cfg.rope_parameters == {+ "full_attention": {"rope_type": "yarn", "factor": 4.0, "rope_theta": 1e6},+ "sliding_attention": {"rope_type": "default", "rope_theta": 1e4},+ }, cfg.rope_parameters++ from unsloth.models.vision import _vllm_rope_scaling++ assert _vllm_rope_scaling(yarn, True) == yarn+ for args in ((new, True), (yarn, False), (yarn, True, True)):+ try:+ _vllm_rope_scaling(*args)+ raise AssertionError(f"vLLM RoPE override accepted: {args}")+ except NotImplementedError:+ pass+ # fast_inference: load_vllm reads config.json itself, so the override must be forwarded.+ import ast, inspect, textwrap+ import unsloth.models.vision as vision++ source = inspect.getsource(vision.FastBaseModel.from_pretrained)+ forwarded = [+ node+ for node in ast.walk(ast.parse(textwrap.dedent(source)))+ if isinstance(node, ast.Assign)+ and any(+ isinstance(t, ast.Subscript)+ and getattr(t.value, "id", None) == "load_vllm_kwargs"+ and getattr(t.slice, "value", None) == "rope_scaling"+ for t in node.targets+ )+ ]+ assert forwarded, "load_vllm_kwargs never receives rope_scaling"+ print("OK", unsloth.__file__)+ sys.exit(0)+yarn = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 40960}+if int(__import__("transformers").__version__.split(".")[0]) >= 5:+ yarn["rope_theta"] = 1000000+cls = FastLanguageModel if loader == "lang" else FastModel+model, _ = cls.from_pretrained(+ # Past the checkpoint's 40960 window, as in the long-context recipe: there the automatic linear+ # scaling used to replace a `rope_parameters` YaRN request.+ "trl-internal-testing/tiny-Qwen3ForCausalLM", max_seq_length = 163840, load_in_4bit = False,+ dtype = torch.float32, device_map = "cpu", **{kwarg: dict(yarn)},+)+cfg = model.config+rope = getattr(cfg, "rope_parameters", None) or getattr(cfg, "rope_scaling", None) or {}+assert rope.get("rope_type") == "yarn", rope+assert rope.get("factor") == 4.0 and rope.get("original_max_position_embeddings") == 40960, rope+rotary = [m for m in model.modules() if "Rotary" in type(m).__name__]+assert rotary, "no rotary module"+expected = 0.1 * math.log(4.0) + 1.0+for r in rotary:+ assert abs(float(r.attention_scaling) - expected) < 1e-9, (type(r), r.attention_scaling)+ if hasattr(r, "rope_type"):+ assert r.rope_type == "yarn", r.rope_type+print("OK", unsloth.__file__)+"""++_V5 = Version(transformers.__version__) >= Version("5.0.0")+++ "loader,kwarg",+ [+ ("lang", "rope_scaling"),+ pytest.param(+ "lang",+ "rope_parameters",+ marks = pytest.mark.skipif(not _V5, reason = "rope_parameters is the v5 spelling"),+ ),+ ("model", "rope_scaling"),+ pytest.param(+ "model",+ "rope_parameters",+ marks = pytest.mark.skipif(not _V5, reason = "rope_parameters is the v5 spelling"),+ ),+ ],+)+def test_yarn_reaches_the_model(loader, kwarg):+ result = subprocess.run(+ [sys.executable, "-c", _CHECK, _TESTS, loader, kwarg],+ capture_output = True,+ text = True,+ timeout = 900,+ )+ assert result.returncode == 0 and "OK" in result.stdout, (+ result.stdout[-2000:] + result.stderr[-4000:]+ )+++def test_rope_helpers():+ result = subprocess.run(+ [sys.executable, "-c", _CHECK, _TESTS, "unit", "-"],+ capture_output = True,+ text = True,+ timeout = 900,+ )+ assert result.returncode == 0 and "OK" in result.stdout, (+ result.stdout[-2000:] + result.stderr[-4000:]+ )diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex adf81493298..9b2d3bb8707 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -225,6 +225,20 @@ global DISABLE_COMPILE_MODEL_NAMES# Must be alphabetically sorted for each entry.+def _fold_rope_parameters(rope_scaling, kwargs):+ """transformers v5 and TRL's model_init_kwargs spell the RoPE override `rope_parameters`.+ Read it as `rope_scaling`, else FastLanguageModel ignores it (and its auto linear scaling+ replaces a YaRN request) and FastModel passes it to the model constructor, which fails."""+ rope_parameters = kwargs.pop("rope_parameters", None)+ if rope_parameters is None:+ return rope_scaling+ if rope_scaling is not None and dict(rope_scaling) != dict(rope_parameters):+ raise ValueError(+ "Unsloth: `rope_scaling` and `rope_parameters` were both given and differ; pass one."+ )+ return rope_parameters++def _strip_unsloth_bnb_4bit_suffix(model_name: str) -> str:"""Remove Unsloth 4bit suffixes without lowercasing (HF cache dirs are case-sensitive)."""s = model_name@@ -1254,6 +1268,7 @@ class FastLanguageModel(FastLlamaModel):on_model_resolved = None,**kwargs,):+ rope_scaling = _fold_rope_parameters(rope_scaling, kwargs)quantization_config = kwargs.get("quantization_config", None)if quantization_config is not None:if isinstance(quantization_config, dict):@@ -1320,7 +1335,7 @@ class FastLanguageModel(FastLlamaModel):token = token,device_map = device_map,device_map_planner_kwargs = device_map_planner_kwargs,- rope_scaling = rope_scaling, # [TODO] No effect+ rope_scaling = rope_scaling,fix_tokenizer = fix_tokenizer,trust_remote_code = trust_remote_code,use_gradient_checkpointing = use_gradient_checkpointing,@@ -1850,7 +1865,7 @@ class FastLanguageModel(FastLlamaModel):token = token,device_map = device_map,device_map_planner_kwargs = device_map_planner_kwargs,- rope_scaling = rope_scaling, # [TODO] No effect+ rope_scaling = rope_scaling,fix_tokenizer = fix_tokenizer,trust_remote_code = trust_remote_code,use_gradient_checkpointing = use_gradient_checkpointing,@@ -2188,7 +2203,7 @@ class FastModel(FastBaseModel):device_map = DEFAULT_DEVICE_MAP,# planner options apply only when device_map = "unsloth".device_map_planner_kwargs = None,- rope_scaling = None, # [TODO] No effect+ rope_scaling = None,fix_tokenizer = True,trust_remote_code = False,use_gradient_checkpointing = "unsloth",@@ -2218,6 +2233,7 @@ class FastModel(FastBaseModel):on_model_resolved = None,**kwargs,):+ rope_scaling = _fold_rope_parameters(rope_scaling, kwargs)user_config = kwargs.pop("config", None)float8_training = resolve_float8_training(kwargs.pop("float8_training", None))if float8_training is not None and not full_finetuning:@@ -3327,6 +3343,7 @@ class FastModel(FastBaseModel):model, tokenizer = FastBaseModel.from_pretrained(model_name = model_name,max_seq_length = max_seq_length,+ rope_scaling = rope_scaling,dtype = _get_dtype(dtype),load_in_4bit = load_in_4bit_kwargs,load_in_8bit = load_in_8bit_kwargs,diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex ac22a9bd2b9..4072b3934db 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -25,6 +25,84 @@ except:from transformers import AutoModelForVision2Seq+# Keys that describe the model's rotary layout, not the scaling: a caller's override keeps them.+_ROPE_LAYOUT_KEYS = ("rope_theta", "partial_rotary_factor", "mrope_section", "mrope_interleaved")+++def _apply_user_rope_scaling(config, rope_scaling):+ """Put the caller's rope_scaling (e.g. YaRN) on the text config before the model is built.+ Returns the dict applied (flat), or None for a per-layer-type layout."""+ from .llama import _rope_scaling_as_dict, _user_rope_scaling++ target = config.get_text_config() if hasattr(config, "get_text_config") else config+ scaling = dict(_rope_scaling_as_dict(rope_scaling))+ nested = bool(scaling) and all(isinstance(v, dict) for v in scaling.values())+ current = getattr(target, "rope_parameters", None)+ # transformers v5 keys rope_parameters by layer type on mixed-attention models (Gemma 3).+ per_layer_type = (+ isinstance(current, dict)+ and current+ and "rope_type" not in current+ and all(isinstance(v, dict) for v in current.values())+ )+ if per_layer_type:+ if not nested and "full_attention" in current:+ # As transformers' Gemma 3 config does: a flat override applies to full attention only.+ scaling = {"full_attention": scaling}+ elif not nested or not set(scaling) <= set(current):+ raise ValueError(+ "Unsloth: this model keeps one RoPE setting per layer type "+ f"({sorted(current)}); pass rope_scaling as a dict keyed by those layer types."+ )+ # Gemma 3's rotary init indexes rope_parameters[layer_type]["rope_theta"]: keep each base.+ target.rope_parameters = {+ k: (+ {**{key: v[key] for key in _ROPE_LAYOUT_KEYS if key in v}, **scaling[k]}+ if k in scaling+ else dict(v)+ )+ for k, v in current.items()+ }+ return None+ if nested:+ raise ValueError(+ "Unsloth: rope_scaling keyed by layer type was given, but this model has one RoPE "+ "setting for every layer; pass a single dict such as {'rope_type': 'yarn', 'factor': 4.0}."+ )+ existing = _rope_scaling_as_dict(+ getattr(target, "rope_parameters", None) or getattr(target, "rope_scaling", None) or {}+ )+ layout = {key: existing[key] for key in _ROPE_LAYOUT_KEYS if key in existing}+ applied = {**layout, **_user_rope_scaling(target, scaling)}+ target.rope_scaling = applied+ return dict(applied)+++def _vllm_rope_scaling(+ rope_scaling,+ load_vllm_takes_it,+ is_vlm = False,+):+ """The caller's RoPE override for load_vllm (flat dicts on text models only)."""+ from .llama import _rope_scaling_as_dict++ scaling = _rope_scaling_as_dict(rope_scaling)+ reason = None+ if not load_vllm_takes_it:+ reason = "this unsloth_zoo's load_vllm has no rope_scaling argument; upgrade unsloth_zoo"+ elif scaling and all(isinstance(v, dict) for v in scaling.values()):+ reason = "one RoPE setting per layer type"+ elif is_vlm:+ # vLLM applies hf_overrides to the top-level config, not the text config a VLM reads RoPE from.+ reason = "vLLM would apply it to the vision-language config, not its text model"+ if reason is not None:+ raise NotImplementedError(+ f"Unsloth: fast_inference = True cannot pass this rope_scaling to vLLM ({reason}), "+ "so generation would not match training. Load with fast_inference = False."+ )+ return dict(scaling)++def _embeddings_or_none(model, getter):"""Call `model.<getter>()`, or None when the model cannot answer.@@ -3270,6 +3348,7 @@ class FastBaseModel:**kwargs,):user_config = kwargs.pop("config", None)+ user_rope_scaling = kwargs.pop("rope_scaling", None)if auto_config is None and user_config is not None:auto_config = user_config_offload_layers = legacy_offload_layers(kwargs, kwargs.pop("offload_layers", None))@@ -4061,6 +4140,20 @@ class FastBaseModel:)_set_attn_impl(auto_config, config_attn_impl)model_config = auto_config+ if user_rope_scaling is not None:+ if fast_inference and is_vLLM_available():+ import inspect as _inspect+ from unsloth_zoo.vllm_utils import load_vllm as _load_vllm+ _vllm_rope_scaling(+ user_rope_scaling,+ "rope_scaling" in _inspect.getfullargspec(_load_vllm).args,+ is_vlm = getattr(model_config, "get_text_config", lambda: model_config)()+ is not model_config,+ )+ # vLLM gets what the model got (theta, original window, layout keys filled in).+ user_rope_scaling = (+ _apply_user_rope_scaling(model_config, user_rope_scaling) or user_rope_scaling+ )verify_fp8_support_if_applicable(model_config, fast_inference = bool(fast_inference and is_vLLM_available())@@ -4324,6 +4417,15 @@ class FastBaseModel:for allowed_arg in allowed_args:if allowed_arg not in load_vllm_kwargs and allowed_arg in kwargs:load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg]+ # vLLM reads config.json itself: the RoPE override must reach it too, else rollouts+ # run the checkpoint's RoPE while training runs the caller's.+ if user_rope_scaling is not None:+ load_vllm_kwargs["rope_scaling"] = _vllm_rope_scaling(+ user_rope_scaling,+ "rope_scaling" in allowed_args,+ is_vlm = getattr(model_config, "get_text_config", lambda: model_config)()+ is not model_config,+ )llm = load_vllm(**load_vllm_kwargs)