Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Honour YaRN passed as rope_parameters, and rope_scaling on FastModel (#15214)

Daniel Han801ba6d049f0
diff --git a/tests/test_optimized_precision_conflicts.py b/tests/test_optimized_precision_conflicts.py
index 034f1355191..5d8908b16f2 100644
--- a/tests/test_optimized_precision_conflicts.py
+++ b/tests/test_optimized_precision_conflicts.py
@@ -59,7 +59,8 @@ def loader():
n
for n in tree.body
if isinstance(n, ast.FunctionDef)
- and n.name in ("_precision_flags_conflict", "_modelscope_snapshot_or_none")
+ and n.name
+ in ("_precision_flags_conflict", "_modelscope_snapshot_or_none", "_fold_rope_parameters")
]
captured = {"config_calls": 0}
diff --git a/tests/test_yarn_rope_parameters_load.py b/tests/test_yarn_rope_parameters_load.py
new file mode 100644
index 00000000000..6c63dad3767
--- /dev/null
+++ b/tests/test_yarn_rope_parameters_load.py
@@ -0,0 +1,217 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""YaRN passed to from_pretrained must reach the model, as `rope_scaling` or as transformers v5 /
+TRL's `rope_parameters` spelling (SFTConfig(model_init_kwargs = {"rope_parameters": ...})).
+
+Before: FastLanguageModel ignored `rope_parameters`, FastModel passed it to the model constructor
+(TypeError), and FastModel dropped `rope_scaling` entirely, so the model trained on plain RoPE."""
+
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from packaging.version import Version
+
+transformers = pytest.importorskip("transformers")
+pytest.importorskip("torch")
+
+_TESTS = str(Path(__file__).resolve().parent)
+
+# Subprocess: the loaders patch transformers process-wide, and the CPU redirects must not leak.
+_CHECK = r"""
+import math, os, sys
+os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")
+os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
+sys.path.insert(0, sys.argv[1])
+import _zoo_aggressive_cuda_spoof as _spoof
+_spoof.apply()
+import torch
+import unsloth
+_is_cuda = lambda d: isinstance(d, (str, int, torch.device)) and not isinstance(d, bool) and torch.device(d).type == "cuda"
+for _name in ("empty", "zeros", "ones", "full", "tensor", "arange", "randn", "rand", "randint",
+ "empty_like", "zeros_like", "ones_like"):
+ _orig = getattr(torch, _name)
+ def _redir(*a, _orig = _orig, **k):
+ if _is_cuda(k.get("device")):
+ k["device"] = "cpu"
+ return _orig(*a, **k)
+ setattr(torch, _name, _redir)
+_orig_to = torch.Tensor.to
+def _to(self, *a, **k):
+ a = tuple("cpu" if _is_cuda(x) else x for x in a)
+ if _is_cuda(k.get("device")):
+ k["device"] = "cpu"
+ return _orig_to(self, *a, **k)
+torch.Tensor.to = _to
+torch.Tensor.cuda = lambda self, *a, **k: self
+if hasattr(torch, "accelerator"):
+ torch.accelerator.is_available = lambda *a, **k: False
+from unsloth import FastLanguageModel, FastModel
+
+loader, kwarg = sys.argv[2], sys.argv[3]
+if loader == "unit":
+ from unsloth.models.loader import _fold_rope_parameters
+ from unsloth.models.vision import _apply_user_rope_scaling
+
+ yarn = {"rope_type": "yarn", "factor": 4.0}
+ kwargs = {"rope_parameters": dict(yarn), "other": 1}
+ assert _fold_rope_parameters(None, kwargs) == yarn
+ assert kwargs == {"other": 1}
+ assert _fold_rope_parameters(dict(yarn), {"rope_parameters": dict(yarn)}) == yarn
+ assert _fold_rope_parameters(yarn, {}) is yarn
+ try:
+ _fold_rope_parameters({"rope_type": "linear", "factor": 2.0}, {"rope_parameters": yarn})
+ raise AssertionError("differing rope_scaling / rope_parameters accepted")
+ except ValueError:
+ pass
+
+ class Cfg:
+ rope_parameters = {
+ "full_attention": {"rope_type": "default", "rope_theta": 1e6},
+ "sliding_attention": {"rope_type": "default", "rope_theta": 1e4},
+ }
+
+ # A flat override applies to full attention only, as transformers' Gemma 3 config does.
+ cfg = Cfg()
+ assert _apply_user_rope_scaling(cfg, {"rope_type": "yarn", "factor": 4.0}) is None
+ assert cfg.rope_parameters == {
+ "full_attention": {"rope_type": "yarn", "factor": 4.0, "rope_theta": 1e6},
+ "sliding_attention": {"rope_type": "default", "rope_theta": 1e4},
+ }, cfg.rope_parameters
+
+ # A flat config keeps its layout keys (Qwen2-VL mrope_section, StableLM partial_rotary_factor).
+ class Flat:
+ max_position_embeddings = 4096
+ rope_parameters = {
+ "rope_type": "default", "rope_theta": 1e4, "partial_rotary_factor": 0.25,
+ "mrope_section": [2, 1, 1],
+ }
+
+ flat = Flat()
+ applied = _apply_user_rope_scaling(flat, {"rope_type": "yarn", "factor": 4.0})
+ expected = {
+ "rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 4096,
+ "rope_theta": 1e4, "partial_rotary_factor": 0.25, "mrope_section": [2, 1, 1],
+ }
+ assert applied == expected and flat.rope_scaling == expected, (applied, flat.rope_scaling)
+ try:
+ _apply_user_rope_scaling(Flat(), {"full_attention": {"rope_type": "yarn", "factor": 4.0}})
+ raise AssertionError("per-layer-type override accepted for a flat config")
+ except ValueError:
+ pass
+ new = {
+ "full_attention": {"rope_type": "yarn", "factor": 4.0, "rope_theta": 1e6},
+ "sliding_attention": {"rope_type": "default", "rope_theta": 1e4},
+ }
+ _apply_user_rope_scaling(cfg, new)
+ assert cfg.rope_parameters == new
+ # Entries without rope_theta keep the layer type's base (Gemma 3's rotary init indexes it).
+ cfg = Cfg()
+ _apply_user_rope_scaling(
+ cfg,
+ {
+ "full_attention": {"rope_type": "yarn", "factor": 4.0},
+ "sliding_attention": {"rope_type": "default"},
+ },
+ )
+ assert cfg.rope_parameters == {
+ "full_attention": {"rope_type": "yarn", "factor": 4.0, "rope_theta": 1e6},
+ "sliding_attention": {"rope_type": "default", "rope_theta": 1e4},
+ }, cfg.rope_parameters
+
+ from unsloth.models.vision import _vllm_rope_scaling
+
+ assert _vllm_rope_scaling(yarn, True) == yarn
+ for args in ((new, True), (yarn, False), (yarn, True, True)):
+ try:
+ _vllm_rope_scaling(*args)
+ raise AssertionError(f"vLLM RoPE override accepted: {args}")
+ except NotImplementedError:
+ pass
+ # fast_inference: load_vllm reads config.json itself, so the override must be forwarded.
+ import ast, inspect, textwrap
+ import unsloth.models.vision as vision
+
+ source = inspect.getsource(vision.FastBaseModel.from_pretrained)
+ forwarded = [
+ node
+ for node in ast.walk(ast.parse(textwrap.dedent(source)))
+ if isinstance(node, ast.Assign)
+ and any(
+ isinstance(t, ast.Subscript)
+ and getattr(t.value, "id", None) == "load_vllm_kwargs"
+ and getattr(t.slice, "value", None) == "rope_scaling"
+ for t in node.targets
+ )
+ ]
+ assert forwarded, "load_vllm_kwargs never receives rope_scaling"
+ print("OK", unsloth.__file__)
+ sys.exit(0)
+yarn = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 40960}
+if int(__import__("transformers").__version__.split(".")[0]) >= 5:
+ yarn["rope_theta"] = 1000000
+cls = FastLanguageModel if loader == "lang" else FastModel
+model, _ = cls.from_pretrained(
+ # Past the checkpoint's 40960 window, as in the long-context recipe: there the automatic linear
+ # scaling used to replace a `rope_parameters` YaRN request.
+ "trl-internal-testing/tiny-Qwen3ForCausalLM", max_seq_length = 163840, load_in_4bit = False,
+ dtype = torch.float32, device_map = "cpu", **{kwarg: dict(yarn)},
+)
+cfg = model.config
+rope = getattr(cfg, "rope_parameters", None) or getattr(cfg, "rope_scaling", None) or {}
+assert rope.get("rope_type") == "yarn", rope
+assert rope.get("factor") == 4.0 and rope.get("original_max_position_embeddings") == 40960, rope
+rotary = [m for m in model.modules() if "Rotary" in type(m).__name__]
+assert rotary, "no rotary module"
+expected = 0.1 * math.log(4.0) + 1.0
+for r in rotary:
+ assert abs(float(r.attention_scaling) - expected) < 1e-9, (type(r), r.attention_scaling)
+ if hasattr(r, "rope_type"):
+ assert r.rope_type == "yarn", r.rope_type
+print("OK", unsloth.__file__)
+"""
+
+_V5 = Version(transformers.__version__) >= Version("5.0.0")
+
+
+ "loader,kwarg",
+ [
+ ("lang", "rope_scaling"),
+ pytest.param(
+ "lang",
+ "rope_parameters",
+ marks = pytest.mark.skipif(not _V5, reason = "rope_parameters is the v5 spelling"),
+ ),
+ ("model", "rope_scaling"),
+ pytest.param(
+ "model",
+ "rope_parameters",
+ marks = pytest.mark.skipif(not _V5, reason = "rope_parameters is the v5 spelling"),
+ ),
+ ],
+)
+def test_yarn_reaches_the_model(loader, kwarg):
+ result = subprocess.run(
+ [sys.executable, "-c", _CHECK, _TESTS, loader, kwarg],
+ capture_output = True,
+ text = True,
+ timeout = 900,
+ )
+ assert result.returncode == 0 and "OK" in result.stdout, (
+ result.stdout[-2000:] + result.stderr[-4000:]
+ )
+
+
+def test_rope_helpers():
+ result = subprocess.run(
+ [sys.executable, "-c", _CHECK, _TESTS, "unit", "-"],
+ capture_output = True,
+ text = True,
+ timeout = 900,
+ )
+ assert result.returncode == 0 and "OK" in result.stdout, (
+ result.stdout[-2000:] + result.stderr[-4000:]
+ )
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index adf81493298..9b2d3bb8707 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -225,6 +225,20 @@ global DISABLE_COMPILE_MODEL_NAMES
# Must be alphabetically sorted for each entry.
+def _fold_rope_parameters(rope_scaling, kwargs):
+ """transformers v5 and TRL's model_init_kwargs spell the RoPE override `rope_parameters`.
+ Read it as `rope_scaling`, else FastLanguageModel ignores it (and its auto linear scaling
+ replaces a YaRN request) and FastModel passes it to the model constructor, which fails."""
+ rope_parameters = kwargs.pop("rope_parameters", None)
+ if rope_parameters is None:
+ return rope_scaling
+ if rope_scaling is not None and dict(rope_scaling) != dict(rope_parameters):
+ raise ValueError(
+ "Unsloth: `rope_scaling` and `rope_parameters` were both given and differ; pass one."
+ )
+ return rope_parameters
+
+
def _strip_unsloth_bnb_4bit_suffix(model_name: str) -> str:
"""Remove Unsloth 4bit suffixes without lowercasing (HF cache dirs are case-sensitive)."""
s = model_name
@@ -1254,6 +1268,7 @@ class FastLanguageModel(FastLlamaModel):
on_model_resolved = None,
**kwargs,
):
+ rope_scaling = _fold_rope_parameters(rope_scaling, kwargs)
quantization_config = kwargs.get("quantization_config", None)
if quantization_config is not None:
if isinstance(quantization_config, dict):
@@ -1320,7 +1335,7 @@ class FastLanguageModel(FastLlamaModel):
token = token,
device_map = device_map,
device_map_planner_kwargs = device_map_planner_kwargs,
- rope_scaling = rope_scaling, # [TODO] No effect
+ rope_scaling = rope_scaling,
fix_tokenizer = fix_tokenizer,
trust_remote_code = trust_remote_code,
use_gradient_checkpointing = use_gradient_checkpointing,
@@ -1850,7 +1865,7 @@ class FastLanguageModel(FastLlamaModel):
token = token,
device_map = device_map,
device_map_planner_kwargs = device_map_planner_kwargs,
- rope_scaling = rope_scaling, # [TODO] No effect
+ rope_scaling = rope_scaling,
fix_tokenizer = fix_tokenizer,
trust_remote_code = trust_remote_code,
use_gradient_checkpointing = use_gradient_checkpointing,
@@ -2188,7 +2203,7 @@ class FastModel(FastBaseModel):
device_map = DEFAULT_DEVICE_MAP,
# planner options apply only when device_map = "unsloth".
device_map_planner_kwargs = None,
- rope_scaling = None, # [TODO] No effect
+ rope_scaling = None,
fix_tokenizer = True,
trust_remote_code = False,
use_gradient_checkpointing = "unsloth",
@@ -2218,6 +2233,7 @@ class FastModel(FastBaseModel):
on_model_resolved = None,
**kwargs,
):
+ rope_scaling = _fold_rope_parameters(rope_scaling, kwargs)
user_config = kwargs.pop("config", None)
float8_training = resolve_float8_training(kwargs.pop("float8_training", None))
if float8_training is not None and not full_finetuning:
@@ -3327,6 +3343,7 @@ class FastModel(FastBaseModel):
model, tokenizer = FastBaseModel.from_pretrained(
model_name = model_name,
max_seq_length = max_seq_length,
+ rope_scaling = rope_scaling,
dtype = _get_dtype(dtype),
load_in_4bit = load_in_4bit_kwargs,
load_in_8bit = load_in_8bit_kwargs,
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index ac22a9bd2b9..4072b3934db 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -25,6 +25,84 @@ except:
from transformers import AutoModelForVision2Seq
+# Keys that describe the model's rotary layout, not the scaling: a caller's override keeps them.
+_ROPE_LAYOUT_KEYS = ("rope_theta", "partial_rotary_factor", "mrope_section", "mrope_interleaved")
+
+
+def _apply_user_rope_scaling(config, rope_scaling):
+ """Put the caller's rope_scaling (e.g. YaRN) on the text config before the model is built.
+ Returns the dict applied (flat), or None for a per-layer-type layout."""
+ from .llama import _rope_scaling_as_dict, _user_rope_scaling
+
+ target = config.get_text_config() if hasattr(config, "get_text_config") else config
+ scaling = dict(_rope_scaling_as_dict(rope_scaling))
+ nested = bool(scaling) and all(isinstance(v, dict) for v in scaling.values())
+ current = getattr(target, "rope_parameters", None)
+ # transformers v5 keys rope_parameters by layer type on mixed-attention models (Gemma 3).
+ per_layer_type = (
+ isinstance(current, dict)
+ and current
+ and "rope_type" not in current
+ and all(isinstance(v, dict) for v in current.values())
+ )
+ if per_layer_type:
+ if not nested and "full_attention" in current:
+ # As transformers' Gemma 3 config does: a flat override applies to full attention only.
+ scaling = {"full_attention": scaling}
+ elif not nested or not set(scaling) <= set(current):
+ raise ValueError(
+ "Unsloth: this model keeps one RoPE setting per layer type "
+ f"({sorted(current)}); pass rope_scaling as a dict keyed by those layer types."
+ )
+ # Gemma 3's rotary init indexes rope_parameters[layer_type]["rope_theta"]: keep each base.
+ target.rope_parameters = {
+ k: (
+ {**{key: v[key] for key in _ROPE_LAYOUT_KEYS if key in v}, **scaling[k]}
+ if k in scaling
+ else dict(v)
+ )
+ for k, v in current.items()
+ }
+ return None
+ if nested:
+ raise ValueError(
+ "Unsloth: rope_scaling keyed by layer type was given, but this model has one RoPE "
+ "setting for every layer; pass a single dict such as {'rope_type': 'yarn', 'factor': 4.0}."
+ )
+ existing = _rope_scaling_as_dict(
+ getattr(target, "rope_parameters", None) or getattr(target, "rope_scaling", None) or {}
+ )
+ layout = {key: existing[key] for key in _ROPE_LAYOUT_KEYS if key in existing}
+ applied = {**layout, **_user_rope_scaling(target, scaling)}
+ target.rope_scaling = applied
+ return dict(applied)
+
+
+def _vllm_rope_scaling(
+ rope_scaling,
+ load_vllm_takes_it,
+ is_vlm = False,
+):
+ """The caller's RoPE override for load_vllm (flat dicts on text models only)."""
+ from .llama import _rope_scaling_as_dict
+
+ scaling = _rope_scaling_as_dict(rope_scaling)
+ reason = None
+ if not load_vllm_takes_it:
+ reason = "this unsloth_zoo's load_vllm has no rope_scaling argument; upgrade unsloth_zoo"
+ elif scaling and all(isinstance(v, dict) for v in scaling.values()):
+ reason = "one RoPE setting per layer type"
+ elif is_vlm:
+ # vLLM applies hf_overrides to the top-level config, not the text config a VLM reads RoPE from.
+ reason = "vLLM would apply it to the vision-language config, not its text model"
+ if reason is not None:
+ raise NotImplementedError(
+ f"Unsloth: fast_inference = True cannot pass this rope_scaling to vLLM ({reason}), "
+ "so generation would not match training. Load with fast_inference = False."
+ )
+ return dict(scaling)
+
+
def _embeddings_or_none(model, getter):
"""Call `model.<getter>()`, or None when the model cannot answer.
@@ -3270,6 +3348,7 @@ class FastBaseModel:
**kwargs,
):
user_config = kwargs.pop("config", None)
+ user_rope_scaling = kwargs.pop("rope_scaling", None)
if auto_config is None and user_config is not None:
auto_config = user_config
_offload_layers = legacy_offload_layers(kwargs, kwargs.pop("offload_layers", None))
@@ -4061,6 +4140,20 @@ class FastBaseModel:
)
_set_attn_impl(auto_config, config_attn_impl)
model_config = auto_config
+ if user_rope_scaling is not None:
+ if fast_inference and is_vLLM_available():
+ import inspect as _inspect
+ from unsloth_zoo.vllm_utils import load_vllm as _load_vllm
+ _vllm_rope_scaling(
+ user_rope_scaling,
+ "rope_scaling" in _inspect.getfullargspec(_load_vllm).args,
+ is_vlm = getattr(model_config, "get_text_config", lambda: model_config)()
+ is not model_config,
+ )
+ # vLLM gets what the model got (theta, original window, layout keys filled in).
+ user_rope_scaling = (
+ _apply_user_rope_scaling(model_config, user_rope_scaling) or user_rope_scaling
+ )
verify_fp8_support_if_applicable(
model_config, fast_inference = bool(fast_inference and is_vLLM_available())
@@ -4324,6 +4417,15 @@ class FastBaseModel:
for allowed_arg in allowed_args:
if allowed_arg not in load_vllm_kwargs and allowed_arg in kwargs:
load_vllm_kwargs[allowed_arg] = kwargs[allowed_arg]
+ # vLLM reads config.json itself: the RoPE override must reach it too, else rollouts
+ # run the checkpoint's RoPE while training runs the caller's.
+ if user_rope_scaling is not None:
+ load_vllm_kwargs["rope_scaling"] = _vllm_rope_scaling(
+ user_rope_scaling,
+ "rope_scaling" in allowed_args,
+ is_vlm = getattr(model_config, "get_text_config", lambda: model_config)()
+ is not model_config,
+ )
llm = load_vllm(**load_vllm_kwargs)