Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Apply an explicit rope_scaling to the model and to vLLM (#13315)
Daniel Han7d2bd0118422
diff --git a/tests/utils/test_rope_scaling_user_arg.py b/tests/utils/test_rope_scaling_user_arg.pynew file mode 100644index 00000000000..3471a545889--- /dev/null+++ b/tests/utils/test_rope_scaling_user_arg.py@@ -0,0 +1,102 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""An explicit `rope_scaling=` reaches the model config and vLLM (issue #3083): it used to be+dropped, so fast_inference ran the checkpoint's RoPE past its window and hit a device assert."""++import pytest+import torch++YARN = {"rope_type": "yarn", "factor": 4.0, "original_max_position_embeddings": 32768}+ROPE_THETA = 1000000.0+++def _qwen2_config():+ from transformers import Qwen2Config+ return Qwen2Config(+ hidden_size = 256,+ intermediate_size = 512,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ max_position_embeddings = 32768,+ rope_theta = ROPE_THETA,+ )+++def test_user_rope_scaling_keeps_rope_theta():+ # Assigning rope_scaling on transformers v5 replaces all of rope_parameters, so without the+ # carried rope_theta the base falls back to 10000.+ from unsloth.models.llama import _get_rope_theta, _user_rope_scaling++ config = _qwen2_config()+ caller = dict(YARN)+ scaling = _user_rope_scaling(config, caller)+ assert caller == YARN, "the caller's dict must not be mutated"+ # vLLM applies it as an hf_override, outside any patched config setter.+ if hasattr(config, "rope_parameters"):+ assert scaling["rope_theta"] == ROPE_THETA+ config.rope_scaling = scaling+ assert _get_rope_theta(config) == ROPE_THETA+ rope = getattr(config, "rope_parameters", None) or config.rope_scaling+ assert rope["rope_type"] == "yarn" and rope["factor"] == 4.0+++def test_user_rope_scaling_fills_rope_type_from_legacy_type():+ from unsloth.models.llama import _user_rope_scaling++ scaling = _user_rope_scaling(_qwen2_config(), {"type": "yarn", "factor": 4.0})+ assert scaling["rope_type"] == "yarn"+ # The checkpoint window, not the widened max_seq_length transformers would fall back to.+ assert scaling["original_max_position_embeddings"] == 32768+++def test_user_rope_scaling_inv_freq_matches_transformers_yarn():+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS++ from unsloth.models.llama import _compute_config_rope_inv_freq, _user_rope_scaling++ config = _qwen2_config()+ config.rope_scaling = _user_rope_scaling(config, YARN)+ rope = getattr(config, "rope_parameters", None) or config.rope_scaling+ inv_freq, attention_scaling = _compute_config_rope_inv_freq(config, rope)+ ref_inv_freq, ref_scaling = ROPE_INIT_FUNCTIONS["yarn"](config, "cpu")+ torch.testing.assert_close(inv_freq.float().cpu(), ref_inv_freq.float(), rtol = 1e-5, atol = 0)+ assert attention_scaling == pytest.approx(ref_scaling) and attention_scaling > 1.0+++def test_yarn_inv_freq_under_meta_device_context():+ # transformers builds the model under a meta device context; yarn's helpers then made meta+ # tensors and the rotary fell back to unscaled RoPE with a warning.+ from unsloth.models.llama import _compute_config_rope_inv_freq++ config = _qwen2_config()+ config.rope_scaling = {**YARN, "rope_theta": ROPE_THETA}+ rope = getattr(config, "rope_parameters", None) or config.rope_scaling+ with torch.device("meta"):+ inv_freq, attention_scaling = _compute_config_rope_inv_freq(config, rope)+ assert inv_freq is not None and inv_freq.device.type == "cpu"+ assert attention_scaling > 1.0+++def test_user_rope_scaling_keeps_an_existing_yarn_window():+ from unsloth.models.llama import _user_rope_scaling++ config = _qwen2_config()+ config.rope_scaling = {**YARN, "rope_theta": ROPE_THETA}+ config.max_position_embeddings = 131072+ scaling = _user_rope_scaling(config, {"rope_type": "yarn", "factor": 8.0})+ assert scaling["original_max_position_embeddings"] == 32768diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 601477952d5..271e0e5983d 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -14,6 +14,7 @@import torchimport gc+import contextlibimport mathimport functoolsfrom typing import Optional, Tuple, List, Union@@ -1856,6 +1857,25 @@ def _rope_scaling_as_dict(rope_scaling):return {}+def _user_rope_scaling(config, rope_scaling):+ """The caller's rope_scaling as a dict. On transformers v5 it carries the config's rope_theta:+ assigning rope_scaling there replaces all of rope_parameters, base included."""+ scaling = dict(_rope_scaling_as_dict(rope_scaling))+ if "rope_type" not in scaling and "type" in scaling:+ scaling["rope_type"] = scaling["type"]+ # Pin YaRN's pretraining window to the checkpoint's, before max_position_embeddings is widened.+ if scaling.get("rope_type") == "yarn" and "original_max_position_embeddings" not in scaling:+ existing = _rope_scaling_as_dict(+ getattr(config, "rope_scaling", None) or getattr(config, "rope_parameters", None) or {}+ )+ scaling["original_max_position_embeddings"] = (+ existing.get("original_max_position_embeddings") or config.max_position_embeddings+ )+ if isinstance(getattr(config, "rope_parameters", None), dict) and "rope_theta" not in scaling:+ scaling["rope_theta"] = _get_rope_theta(config)+ return scaling++def _extended_rope_scaling(config, factor):"""RoPE scaling to extend a model past its native window. Keeps native llama3 as-is(linear extension is far worse for long context); everything else gets linear. Returns@@ -1932,17 +1952,24 @@ def _compute_config_rope_inv_freq(config, rope_scaling):from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONSrope_init_fn = ROPE_INIT_FUNCTIONS[rope_type]- try:- inv_freq, attention_scaling = rope_init_fn(config, torch.device("cpu"))- except Exception:- # Object-style rope_scaling: retry with a config copy carrying the plain dict.- if isinstance(original_rope_scaling, dict):- raise- import copy as _copy-- config_copy = _copy.copy(config)- config_copy.rope_scaling = rope_scaling- inv_freq, attention_scaling = rope_init_fn(config_copy, torch.device("cpu"))+ # Under the meta-device load context yarn's inner aranges land on meta and the copy to CPU fails;+ # the device mode is only entered then, as it slows every op inside it.+ with (+ torch.device("cpu")+ if torch.get_default_device().type != "cpu"+ else contextlib.nullcontext()+ ):+ try:+ inv_freq, attention_scaling = rope_init_fn(config, torch.device("cpu"))+ except Exception:+ # Object-style rope_scaling: retry with a config copy carrying the plain dict.+ if isinstance(original_rope_scaling, dict):+ raise+ import copy as _copy++ config_copy = _copy.copy(config)+ config_copy.rope_scaling = rope_scaling+ inv_freq, attention_scaling = rope_init_fn(config_copy, torch.device("cpu"))return inv_freq.to(dtype = torch.float32, device = "cpu"), float(attention_scaling)except Exception as exception:if rope_type == "llama3":@@ -2936,12 +2963,36 @@ class FastLlamaModel:if max_seq_length is None:max_seq_length = model_max_seq_length- if (rope_scaling is None) and (max_seq_length > model_max_seq_length):+ caller_rope_scaling = rope_scaling is not None+ if caller_rope_scaling:+ _rope = _rope_scaling_as_dict(rope_scaling)+ _rope_type = _rope.get("rope_type") or _rope.get("type")+ # Only static types Unsloth's rotary reproduces exactly (no seq_len-dependent dynamic or+ # longrope switch; Gemma's rotary ignores config scaling), else vLLM and training disagree.+ if model_config.model_type in ("gemma", "gemma2") or _rope_type not in (+ "linear",+ "yarn",+ "llama3",+ ):+ logger.warning_once(+ f"Unsloth: `rope_scaling` of type {_rope_type} is not supported for "+ f"{model_config.model_type} and is ignored."+ )+ rope_scaling = None+ if rope_scaling is not None:+ # On the config for the in-process load and in kwargs for load_vllm, else vLLM keeps the+ # checkpoint's RoPE and indexes past its cos/sin cache (#3083).+ rope_scaling = _user_rope_scaling(model_config, rope_scaling)+ model_config.rope_scaling = rope_scaling+ kwargs["rope_scaling"] = rope_scaling+ elif not caller_rope_scaling and max_seq_length > model_max_seq_length:factor = max_seq_length / model_max_seq_lengthif fast_inference:raise NotImplementedError(- "Unsloth: Fast inference does not yet work with RoPE Scaling."+ "Unsloth: Fast inference does not yet work with automatic RoPE Scaling.\n"+ f"{model_name} supports {model_max_seq_length} tokens; pass `rope_scaling` "+ f"explicitly (for example YaRN) to use max_seq_length = {max_seq_length}.")linear_scaling, native_type = _extended_rope_scaling(model_config, factor)@@ -3424,6 +3475,9 @@ class FastLlamaModel:model = convert_vllm_to_huggingface(quant_state_dict, model_config, dtype, bnb_config)+ # The rebuilt model takes the checkpoint's config, so put the caller's RoPE back for saving.+ if rope_scaling is not None:+ model.config.rope_scaling = rope_scalingmodel.vllm_engine = llmllm.shared_weights = Truemodel.fast_generate = model.vllm_engine.generate