Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Read plain RoPE as rope_scaling None for 4.x-era remote configs on transformers 5 (#11611)

Daniel Han0065fade6b32
diff --git a/tests/test_remote_rope_scaling_none.py b/tests/test_remote_rope_scaling_none.py
new file mode 100644
index 00000000000..4cce09c6b96
--- /dev/null
+++ b/tests/test_remote_rope_scaling_none.py
@@ -0,0 +1,177 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Remote modeling code written for transformers 4.x reads ``config.rope_scaling is None`` as plain RoPE.
+
+transformers 5 made ``rope_scaling`` an alias of ``rope_parameters``, which is a dict even for plain
+RoPE, so inclusionAI/Ling-2.6-flash's attention read scaling keys it never has. Remote configs with
+plain RoPE read ``None`` again; native configs and real scaling dicts are unchanged.
+"""
+
+import pytest
+import torch
+import transformers
+
+import unsloth
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
+
+# On 4.x the None rope_scaling is native and the shim is a no-op.
+only_v5 = pytest.mark.skipif(
+ int(transformers.__version__.split(".")[0]) < 5, reason = "transformers 4.x needs no shim"
+)
+
+
+@only_v5
+def test_the_shared_rope_table_is_not_given_a_default():
+ """transformers 5.17's `_init_weights` spreads ROPE_INIT_FUNCTIONS over a rotary's own default,
+ so a global "default" entry would replace every native model's own. Remote modules get theirs
+ from their own namespace instead."""
+ assert "default" not in ROPE_INIT_FUNCTIONS
+
+
+def _config_class(
+ module_name,
+ rope_parameters,
+ legacy = True,
+):
+ from transformers import PretrainedConfig
+
+ if legacy: # written for 4.x: takes rope_scaling
+
+ def __init__(
+ self,
+ rope_scaling = None,
+ **kwargs,
+ ):
+ PretrainedConfig.__init__(self, **kwargs)
+
+ else: # written for 5.x: takes rope_parameters
+
+ def __init__(
+ self,
+ rope_parameters = None,
+ **kwargs,
+ ):
+ PretrainedConfig.__init__(self, **kwargs)
+
+ cls = type(
+ "RemoteConfig",
+ (PretrainedConfig,),
+ {"model_type": "remote_plain_rope_test", "__init__": __init__},
+ )
+ cls.__module__ = module_name
+ config = cls()
+ config.rope_parameters = rope_parameters
+ return config
+
+
+@only_v5
+def test_remote_config_with_plain_rope_reads_rope_scaling_none():
+ plain = {"rope_type": "default", "rope_theta": 6_000_000}
+ remote = _config_class(
+ "transformers_modules.inclusionAI.Ling.configuration_bailing", dict(plain)
+ )
+ assert remote.rope_scaling is None
+ assert remote.rope_parameters == plain # the real dict is still there
+
+ yarn = {"rope_type": "yarn", "factor": 4.0, "rope_theta": 1e6}
+ remote_yarn = _config_class("transformers_modules.x.configuration_x", dict(yarn))
+ assert remote_yarn.rope_scaling == yarn
+
+ native = _config_class("transformers.models.llama.configuration_llama", dict(plain))
+ assert native.rope_scaling == plain
+
+ # A remote config written for 5.x keeps the alias.
+ v5 = _config_class("transformers_modules.x.configuration_x", dict(plain), legacy = False)
+ assert v5.rope_scaling == plain
+
+
+@only_v5
+def test_a_native_default_rope_is_not_replaced_on_load(tmp_path):
+ """ERNIE-4.5-VL pre-rotates its default inv_freq; loading must keep the model's own."""
+ ernie = pytest.importorskip("transformers.models.ernie4_5_vl_moe.modeling_ernie4_5_vl_moe")
+ from transformers import AutoConfig
+
+ try:
+ config = AutoConfig.for_model(
+ "ernie4_5_vl_moe",
+ text_config = dict(
+ hidden_size = 64,
+ intermediate_size = 64,
+ moe_intermediate_size = [32, 32],
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ moe_num_experts = 4,
+ moe_k = 2,
+ vocab_size = 128,
+ moe_num_shared_experts = 1,
+ mlp_layer_types = ["dense", "sparse"],
+ rope_parameters = {
+ "rope_type": "default",
+ "rope_theta": 500000.0,
+ "mrope_section": [2, 2, 4],
+ },
+ ),
+ vision_config = dict(depth = 1, hidden_size = 32, intermediate_size = 32, num_heads = 2),
+ )
+ except Exception as error: # 5.4's strict text config rejects its own use_bias default
+ pytest.skip(f"this transformers cannot build the ERNIE-4.5-VL config: {error}")
+ model = ernie.Ernie4_5_VLMoeModel(config)
+ model.save_pretrained(tmp_path)
+ loaded = ernie.Ernie4_5_VLMoeModel.from_pretrained(tmp_path)
+ rotary = loaded.language_model.rotary_emb
+ expected, _ = type(rotary).compute_default_rope_parameters(rotary.config)
+ torch.testing.assert_close(rotary.inv_freq.float(), expected.float(), rtol = 0, atol = 0)
+
+
+@only_v5
+def test_remote_scaling_dict_does_not_repeat_the_base_its_config_keeps():
+ # InternLM2's 4.x config keeps rope_theta as an attribute and rejects any rope_scaling that is
+ # not exactly {type, factor}; transformers 5 and the theta carry put the base in the dict too.
+ from transformers import PretrainedConfig
+
+ class InternLM2LikeConfig(PretrainedConfig):
+ model_type = "remote_internlm2_like_test"
+
+ def __init__(
+ self,
+ rope_theta = 1_000_000,
+ rope_scaling = None,
+ **kwargs,
+ ):
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ if self.rope_scaling is not None and len(self.rope_scaling) != 2:
+ raise ValueError(f"`rope_scaling` must have two fields, got {self.rope_scaling}")
+ super().__init__(**kwargs)
+
+ InternLM2LikeConfig.__module__ = "transformers_modules.internlm.configuration_internlm2"
+ config = InternLM2LikeConfig(rope_scaling = {"type": "dynamic", "factor": 2.0})
+ assert config.rope_scaling["type"] == "dynamic" and config.rope_scaling["factor"] == 2.0
+ assert "rope_theta" not in config.rope_scaling
+ assert config.rope_parameters["rope_theta"] == 1_000_000 # still where transformers 5 reads it
+
+
+ getattr(unsloth, "_IS_MLX", False), reason = "the MLX path leaves it to unsloth_zoo.mlx.loader"
+)
+def test_is_torch_fx_available_imports_as_in_4x():
+ # Ling / BailingMoe, DeepSeek-V3 and Kimi-K2 remote code import it at module level.
+ from transformers.utils.import_utils import is_torch_available, is_torch_fx_available
+ assert is_torch_fx_available() == is_torch_available()
+
+
+def test_rope_scaling_none_also_runs_on_the_mlx_branch():
+ # Apple Silicon never reaches _gpu_init, and remote configs are built there too.
+ import ast
+ from pathlib import Path
+
+ source = Path(unsloth.__file__).read_text(encoding = "utf-8")
+ branch = next(
+ node
+ for node in ast.parse(source).body
+ if isinstance(node, ast.If) and ast.unparse(node.test) == "_IS_MLX"
+ )
+ body = ast.unparse(ast.Module(body = branch.body, type_ignores = []))
+ assert "fix_transformers_remote_rope_scaling_none" in body
diff --git a/unsloth/__init__.py b/unsloth/__init__.py
index d1f0a89e891..5c1c15f8f9d 100644
--- a/unsloth/__init__.py
+++ b/unsloth/__init__.py
@@ -199,6 +199,16 @@ if _IS_MLX:
del _fix_image_reexports
except Exception:
pass
+ try:
+ # Same reason: 4.x remote configs are built here too, and their validators read plain RoPE
+ # as rope_scaling None. is_torch_fx_available is left to unsloth_zoo.mlx.loader.
+ from .import_fixes import (
+ fix_transformers_remote_rope_scaling_none as _fix_remote_rope_scaling,
+ )
+ _fix_remote_rope_scaling()
+ del _fix_remote_rope_scaling
+ except Exception:
+ pass
try:
import unsloth_zoo
except ImportError as _e:
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index f64963526cb..917d1436c87 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -249,6 +249,8 @@ from .import_fixes import (
fix_transformers_composite_prefix_renaming,
fix_transformers_fully_masked_rows,
fix_transformers_rope_scaling_drops_theta,
+ fix_transformers_remote_rope_scaling_none,
+ fix_transformers_is_torch_fx_available,
fix_xformers_performance_issue,
fix_flash_attn_4_namespace_shadow,
fix_vllm_aimv2_issue,
@@ -307,6 +309,9 @@ del check_transformers_prequantized_vlm_quant_state
# RoPE base frequency. Ordered here, before any config is built, so the object-style delegation
# retry in models/llama.py sees a config that kept its base (#2405).
fix_transformers_rope_scaling_drops_theta()
+# Remote code written for 4.x reads plain RoPE as rope_scaling None and imports is_torch_fx_available.
+fix_transformers_remote_rope_scaling_none()
+fix_transformers_is_torch_fx_available()
# Probe-gated and lazy: only wraps get_class_in_module, so the siglip image
# modules are imported and patched when a checkpoint's own modeling file runs,
# not on every `import unsloth`.
@@ -362,6 +367,8 @@ patch_accelerate_recursively_apply()
del fix_transformers5_bare_annotation_configs
del fix_transformers_rope_scaling_drops_theta
+del fix_transformers_remote_rope_scaling_none
+del fix_transformers_is_torch_fx_available
del fix_xformers_performance_issue
del fix_flash_attn_4_namespace_shadow
del fix_vllm_aimv2_issue
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index bed79e718a2..f60639182e9 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -2313,6 +2313,88 @@ def _transformers_rope_scaling_assignment_drops_theta():
return False
+_PLAIN_ROPE_KEYS = frozenset({"rope_type", "type", "rope_theta", "partial_rotary_factor"})
+_ATTRIBUTE_ROPE_KEYS = ("rope_theta", "partial_rotary_factor")
+_NO_ROPE_ATTRIBUTE = object()
+
+
+def fix_transformers_remote_rope_scaling_none():
+ """For remote-code configs only, read plain RoPE back as ``rope_scaling = None`` as 4.x did.
+
+ transformers 5 aliases ``rope_scaling`` to ``rope_parameters``, which is never None, so hub code
+ guarded by ``if config.rope_scaling is not None`` indexes missing keys (``KeyError: 'factor'``).
+ Only configs whose ``__init__`` takes ``rope_scaling`` and not ``rope_parameters``; native and
+ 5.x-authored configs are untouched; a real scaling dict only loses the base and partial rotary
+ factor it duplicates from the config's own attributes."""
+ try:
+ from transformers.configuration_utils import PretrainedConfig
+ except Exception:
+ return
+ prop = PretrainedConfig.__dict__.get("rope_scaling")
+ if not isinstance(prop, property) or prop.fget is None:
+ return
+ if getattr(prop.fget, "_unsloth_remote_plain_rope_none", False):
+ return
+ original_get = prop.fget
+
+ legacy = {}
+
+ def _written_for_4x(cls):
+ # A 4.x-era config takes rope_scaling in __init__; a 5.x one takes rope_parameters.
+ if cls not in legacy:
+ try:
+ parameters = inspect.signature(cls.__init__).parameters
+ legacy[cls] = "rope_scaling" in parameters and "rope_parameters" not in parameters
+ except (TypeError, ValueError):
+ legacy[cls] = False
+ return legacy[cls]
+
+ @functools.wraps(original_get)
+ def rope_scaling(self):
+ value = original_get(self)
+ if not (
+ isinstance(value, dict)
+ and "transformers_modules" in (type(self).__module__ or "")
+ and _written_for_4x(type(self))
+ ):
+ return value
+ if (
+ value.get("rope_type", value.get("type", "default")) == "default"
+ and set(value) <= _PLAIN_ROPE_KEYS
+ ):
+ return None
+ # 4.x kept these as config attributes; their copies in a real scaling dict fail 4.x
+ # validators that expect exactly the checkpoint's keys (InternLM2: len(rope_scaling) == 2).
+ moved = [
+ key
+ for key in _ATTRIBUTE_ROPE_KEYS
+ if key in value and self.__dict__.get(key, _NO_ROPE_ATTRIBUTE) == value[key]
+ ]
+ if not moved:
+ return value
+ return {key: item for key, item in value.items() if key not in moved}
+
+ rope_scaling._unsloth_remote_plain_rope_none = True
+ PretrainedConfig.rope_scaling = property(rope_scaling, prop.fset, prop.fdel, prop.__doc__)
+
+
+def fix_transformers_is_torch_fx_available():
+ """Restore ``transformers.utils.import_utils.is_torch_fx_available``, removed in 5.x and
+ imported at module level by 4.x-era remote code (Ling / BailingMoe, DeepSeek-V3, Kimi-K2).
+ Same body as 4.57.6; only added when missing."""
+ try:
+ import transformers.utils.import_utils as import_utils
+ except Exception:
+ return
+ if hasattr(import_utils, "is_torch_fx_available"):
+ return
+
+ def is_torch_fx_available():
+ return import_utils.is_torch_available()
+
+ import_utils.is_torch_fx_available = is_torch_fx_available
+
+
def fix_transformers_rope_scaling_drops_theta():
"""Stop a replaced ``rope_scaling`` silently unscaling RoPE (issue #2405).