Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Apply an explicit rope_scaling to the model and to vLLM (#13315)

Daniel Han7d2bd0118422
diff --git a/tests/utils/test_rope_scaling_user_arg.py b/tests/utils/test_rope_scaling_user_arg.py
new file mode 100644
index 00000000000..3471a545889
--- /dev/null
+++ b/tests/utils/test_rope_scaling_user_arg.py
@@ -0,0 +1,102 @@
+# Unsloth Zoo - Utilities for Unsloth
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""An explicit `rope_scaling=` reaches the model config and vLLM (issue #3083): it used to be
+dropped, so fast_inference ran the checkpoint's RoPE past its window and hit a device assert."""
+
+import pytest
+import torch
+
+YARN = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 32768}
+ROPE_THETA = 1000000.0
+
+
+def _qwen2_config():
+ from transformers import Qwen2Config
+ return Qwen2Config(
+ hidden_size = 256,
+ intermediate_size = 512,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ max_position_embeddings = 32768,
+ rope_theta = ROPE_THETA,
+ )
+
+
+def test_user_rope_scaling_keeps_rope_theta():
+ # Assigning rope_scaling on transformers v5 replaces all of rope_parameters, so without the
+ # carried rope_theta the base falls back to 10000.
+ from unsloth.models.llama import _get_rope_theta, _user_rope_scaling
+
+ config = _qwen2_config()
+ caller = dict(YARN)
+ scaling = _user_rope_scaling(config, caller)
+ assert caller == YARN, "the caller's dict must not be mutated"
+ # vLLM applies it as an hf_override, outside any patched config setter.
+ if hasattr(config, "rope_parameters"):
+ assert scaling["rope_theta"] == ROPE_THETA
+ config.rope_scaling = scaling
+ assert _get_rope_theta(config) == ROPE_THETA
+ rope = getattr(config, "rope_parameters", None) or config.rope_scaling
+ assert rope["rope_type"] == "yarn" and rope["factor"] == 4.0
+
+
+def test_user_rope_scaling_fills_rope_type_from_legacy_type():
+ from unsloth.models.llama import _user_rope_scaling
+
+ scaling = _user_rope_scaling(_qwen2_config(), {"type": "yarn", "factor": 4.0})
+ assert scaling["rope_type"] == "yarn"
+ # The checkpoint window, not the widened max_seq_length transformers would fall back to.
+ assert scaling["original_max_position_embeddings"] == 32768
+
+
+def test_user_rope_scaling_inv_freq_matches_transformers_yarn():
+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
+
+ from unsloth.models.llama import _compute_config_rope_inv_freq, _user_rope_scaling
+
+ config = _qwen2_config()
+ config.rope_scaling = _user_rope_scaling(config, YARN)
+ rope = getattr(config, "rope_parameters", None) or config.rope_scaling
+ inv_freq, attention_scaling = _compute_config_rope_inv_freq(config, rope)
+ ref_inv_freq, ref_scaling = ROPE_INIT_FUNCTIONS["yarn"](config, "cpu")
+ torch.testing.assert_close(inv_freq.float().cpu(), ref_inv_freq.float(), rtol = 1e-5, atol = 0)
+ assert attention_scaling == pytest.approx(ref_scaling) and attention_scaling > 1.0
+
+
+def test_yarn_inv_freq_under_meta_device_context():
+ # transformers builds the model under a meta device context; yarn's helpers then made meta
+ # tensors and the rotary fell back to unscaled RoPE with a warning.
+ from unsloth.models.llama import _compute_config_rope_inv_freq
+
+ config = _qwen2_config()
+ config.rope_scaling = {**YARN, "rope_theta": ROPE_THETA}
+ rope = getattr(config, "rope_parameters", None) or config.rope_scaling
+ with torch.device("meta"):
+ inv_freq, attention_scaling = _compute_config_rope_inv_freq(config, rope)
+ assert inv_freq is not None and inv_freq.device.type == "cpu"
+ assert attention_scaling > 1.0
+
+
+def test_user_rope_scaling_keeps_an_existing_yarn_window():
+ from unsloth.models.llama import _user_rope_scaling
+
+ config = _qwen2_config()
+ config.rope_scaling = {**YARN, "rope_theta": ROPE_THETA}
+ config.max_position_embeddings = 131072
+ scaling = _user_rope_scaling(config, {"rope_type": "yarn", "factor": 8.0})
+ assert scaling["original_max_position_embeddings"] == 32768
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 601477952d5..271e0e5983d 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -14,6 +14,7 @@
import torch
import gc
+import contextlib
import math
import functools
from typing import Optional, Tuple, List, Union
@@ -1856,6 +1857,25 @@ def _rope_scaling_as_dict(rope_scaling):
return {}
+def _user_rope_scaling(config, rope_scaling):
+ """The caller's rope_scaling as a dict. On transformers v5 it carries the config's rope_theta:
+ assigning rope_scaling there replaces all of rope_parameters, base included."""
+ scaling = dict(_rope_scaling_as_dict(rope_scaling))
+ if "rope_type" not in scaling and "type" in scaling:
+ scaling["rope_type"] = scaling["type"]
+ # Pin YaRN's pretraining window to the checkpoint's, before max_position_embeddings is widened.
+ if scaling.get("rope_type") == "yarn" and "original_max_position_embeddings" not in scaling:
+ existing = _rope_scaling_as_dict(
+ getattr(config, "rope_scaling", None) or getattr(config, "rope_parameters", None) or {}
+ )
+ scaling["original_max_position_embeddings"] = (
+ existing.get("original_max_position_embeddings") or config.max_position_embeddings
+ )
+ if isinstance(getattr(config, "rope_parameters", None), dict) and "rope_theta" not in scaling:
+ scaling["rope_theta"] = _get_rope_theta(config)
+ return scaling
+
+
def _extended_rope_scaling(config, factor):
"""RoPE scaling to extend a model past its native window. Keeps native llama3 as-is
(linear extension is far worse for long context); everything else gets linear. Returns
@@ -1932,17 +1952,24 @@ def _compute_config_rope_inv_freq(config, rope_scaling):
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
rope_init_fn = ROPE_INIT_FUNCTIONS[rope_type]
- try:
- inv_freq, attention_scaling = rope_init_fn(config, torch.device("cpu"))
- except Exception:
- # Object-style rope_scaling: retry with a config copy carrying the plain dict.
- if isinstance(original_rope_scaling, dict):
- raise
- import copy as _copy
-
- config_copy = _copy.copy(config)
- config_copy.rope_scaling = rope_scaling
- inv_freq, attention_scaling = rope_init_fn(config_copy, torch.device("cpu"))
+ # Under the meta-device load context yarn's inner aranges land on meta and the copy to CPU fails;
+ # the device mode is only entered then, as it slows every op inside it.
+ with (
+ torch.device("cpu")
+ if torch.get_default_device().type != "cpu"
+ else contextlib.nullcontext()
+ ):
+ try:
+ inv_freq, attention_scaling = rope_init_fn(config, torch.device("cpu"))
+ except Exception:
+ # Object-style rope_scaling: retry with a config copy carrying the plain dict.
+ if isinstance(original_rope_scaling, dict):
+ raise
+ import copy as _copy
+
+ config_copy = _copy.copy(config)
+ config_copy.rope_scaling = rope_scaling
+ inv_freq, attention_scaling = rope_init_fn(config_copy, torch.device("cpu"))
return inv_freq.to(dtype = torch.float32, device = "cpu"), float(attention_scaling)
except Exception as exception:
if rope_type == "llama3":
@@ -2936,12 +2963,36 @@ class FastLlamaModel:
if max_seq_length is None:
max_seq_length = model_max_seq_length
- if (rope_scaling is None) and (max_seq_length > model_max_seq_length):
+ caller_rope_scaling = rope_scaling is not None
+ if caller_rope_scaling:
+ _rope = _rope_scaling_as_dict(rope_scaling)
+ _rope_type = _rope.get("rope_type") or _rope.get("type")
+ # Only static types Unsloth's rotary reproduces exactly (no seq_len-dependent dynamic or
+ # longrope switch; Gemma's rotary ignores config scaling), else vLLM and training disagree.
+ if model_config.model_type in ("gemma", "gemma2") or _rope_type not in (
+ "linear",
+ "yarn",
+ "llama3",
+ ):
+ logger.warning_once(
+ f"Unsloth: `rope_scaling` of type {_rope_type} is not supported for "
+ f"{model_config.model_type} and is ignored."
+ )
+ rope_scaling = None
+ if rope_scaling is not None:
+ # On the config for the in-process load and in kwargs for load_vllm, else vLLM keeps the
+ # checkpoint's RoPE and indexes past its cos/sin cache (#3083).
+ rope_scaling = _user_rope_scaling(model_config, rope_scaling)
+ model_config.rope_scaling = rope_scaling
+ kwargs["rope_scaling"] = rope_scaling
+ elif not caller_rope_scaling and max_seq_length > model_max_seq_length:
factor = max_seq_length / model_max_seq_length
if fast_inference:
raise NotImplementedError(
- "Unsloth: Fast inference does not yet work with RoPE Scaling."
+ "Unsloth: Fast inference does not yet work with automatic RoPE Scaling.\n"
+ f"{model_name} supports {model_max_seq_length} tokens; pass `rope_scaling` "
+ f"explicitly (for example YaRN) to use max_seq_length = {max_seq_length}."
)
linear_scaling, native_type = _extended_rope_scaling(model_config, factor)
@@ -3424,6 +3475,9 @@ class FastLlamaModel:
model = convert_vllm_to_huggingface(
quant_state_dict, model_config, dtype, bnb_config
)
+ # The rebuilt model takes the checkpoint's config, so put the caller's RoPE back for saving.
+ if rope_scaling is not None:
+ model.config.rope_scaling = rope_scaling
model.vllm_engine = llm
llm.shared_weights = True
model.fast_generate = model.vllm_engine.generate