Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep every nested RoPE base, and never put a per-label mapping in rope_theta (#11192)

Daniel Han14be511f4ed3
diff --git a/tests/test_import_fixes_drift.py b/tests/test_import_fixes_drift.py
index 63b7de6981d..c175130b402 100644
--- a/tests/test_import_fixes_drift.py
+++ b/tests/test_import_fixes_drift.py
@@ -1213,6 +1213,299 @@ def test_rope_theta_carry_only_writes_when_the_base_would_be_lost():
assert parameters == {"rope_type": "default"}
+# The two shapes the carry got wrong when it first landed (#11037).
+
+T5GEMMA2_LAYER_TYPES = ["sliding_attention", "sliding_attention", "full_attention"]
+T5GEMMA2_ROPE = {
+ "sliding_attention": {"rope_type": "default", "rope_theta": 10000.0},
+ "full_attention": {"rope_type": "default", "rope_theta": 1000000.0},
+}
+DEEPSEEK_V4_LABELS = ("main", "compress")
+DEEPSEEK_V4_ROPE = {
+ "main": {"rope_type": "default", "rope_theta": 10000.0, "partial_rotary_factor": 0.125},
+ "compress": {"rope_type": "default", "rope_theta": 160000.0, "partial_rotary_factor": 0.125},
+}
+
+
+def test_rope_theta_carry_never_puts_a_per_label_mapping_in_the_scalar_slot():
+ """``_carry_per_layer_rope_theta`` returns ``None`` for the no-op too, and #11037 as merged
+ read that as "fall through", carrying the mapping into the scalar ``config.rope_theta``."""
+ from types import SimpleNamespace
+
+ from unsloth.import_fixes import _carry_rope_theta_across_assignment as carry
+
+ parameters = {k: dict(v) for k, v in T5GEMMA2_ROPE.items()}
+ config = SimpleNamespace(
+ rope_parameters = parameters,
+ layer_types = list(T5GEMMA2_LAYER_TYPES),
+ )
+ carried = {"sliding_attention": 10000.0, "full_attention": 1000000.0}
+
+ assert (
+ carry(config, carried) is None
+ ), "the carry reported it carried a base across an assignment that lost nothing"
+ assert not hasattr(config, "rope_theta"), (
+ f"a per-label MAPPING reached the scalar rope_theta slot: "
+ f"{getattr(config, 'rope_theta', None)!r}. A base frequency is a number, and a "
+ f"config that had no rope_theta must not gain one from a no-op assignment."
+ )
+ assert config.rope_parameters == {
+ k: dict(v) for k, v in T5GEMMA2_ROPE.items()
+ }, "the nested parameters were rewritten by an assignment that lost nothing"
+
+ # Same refusal when the nested write is REFUSED rather than a no-op.
+ stubborn = {
+ "sliding_attention": {"rope_type": "default"},
+ "full_attention": {"rope_type": "default"},
+ }
+
+ class _Frozen(SimpleNamespace):
+ def __setattr__(self, name, value):
+ if name == "rope_parameters" and getattr(self, "_locked", False):
+ raise AttributeError("read-only")
+ super().__setattr__(name, value)
+
+ frozen = _Frozen(rope_parameters = stubborn, layer_types = list(T5GEMMA2_LAYER_TYPES))
+ frozen._locked = True
+ assert carry(frozen, carried) is None
+ assert not isinstance(
+ getattr(frozen, "rope_theta", None), dict
+ ), "a refused nested write fell through and put the mapping in the scalar slot"
+
+
+def test_rope_theta_carry_follows_rope_type_labels_not_only_layer_types():
+ """``standardize_rope_params`` resolves the nesting axis as ``getattr(self,
+ "_rope_type_labels", getattr(self, "layer_types", None))`` (transformers 5.17.0
+ ``modeling_rope_utils.py``); on DeepseekV4 the two name different things."""
+ from types import SimpleNamespace
+
+ from unsloth.import_fixes import (
+ _carry_rope_theta_across_assignment as carry,
+ _rope_parameters_are_per_layer,
+ _rope_theta_snapshot,
+ )
+
+ parameters = {k: dict(v) for k, v in DEEPSEEK_V4_ROPE.items()}
+ config = SimpleNamespace(
+ _rope_type_labels = DEEPSEEK_V4_LABELS,
+ layer_types = ["heavily_compressed_attention", "compressed_sparse_attention"],
+ rope_theta = 10000.0,
+ rope_parameters = parameters,
+ )
+
+ assert _rope_parameters_are_per_layer(
+ config, parameters
+ ), "rope keyed by _rope_type_labels was read as a flat dict"
+ assert _rope_theta_snapshot(config) == {
+ "main": 10000.0,
+ "compress": 160000.0,
+ }, f"the per-label bases were not snapshotted: {_rope_theta_snapshot(config)!r}"
+ carried = _rope_theta_snapshot(config)
+
+ config.rope_parameters = {
+ "main": {"rope_type": "linear", "factor": 4.0},
+ "compress": {"rope_type": "default"},
+ }
+ carry(config, carried)
+
+ restored = config.rope_parameters
+ assert "rope_theta" not in restored, (
+ f"a top-level rope_theta was written into a NESTED rope dict: {restored!r}. "
+ f"transformers would hand that one global base to every label."
+ )
+ assert restored["compress"]["rope_theta"] == 160000.0, (
+ f"the compression base was lost: {restored['compress'].get('rope_theta')!r} "
+ f"(expected 160000.0). standardize_rope_params would setdefault the 10000.0 "
+ f"global base into it instead."
+ )
+ assert restored["main"]["rope_theta"] == 10000.0
+ assert restored["main"]["factor"] == 4.0, "the caller's scaling was damaged"
+ assert config.rope_theta == 10000.0, "the stated global base went stale"
+
+ # Flat replacement, same config: the two bases disagree, so the config's own base is used.
+ flat = SimpleNamespace(
+ _rope_type_labels = DEEPSEEK_V4_LABELS,
+ rope_theta = 10000.0,
+ rope_parameters = {"rope_type": "linear", "factor": 4.0},
+ )
+ assert carry(flat, {"main": 10000.0, "compress": 160000.0}) == 10000.0
+ assert flat.rope_parameters["rope_theta"] == 10000.0
+
+
+def test_rope_carry_handles_rope_labels_not_all_present_in_layer_types():
+ """transformers asks ``isdisjoint``, not ``issubset``: laguna, mellum and zaya ship a rope
+ dict naming a label their default ``layer_types`` omits. Under the subset test all their
+ bases went to ``None`` -- the ``TypeError: ... 'NoneType' and 'Tensor'`` #11037 prevents."""
+ from types import SimpleNamespace
+
+ from unsloth.import_fixes import (
+ _carry_rope_theta_across_assignment as carry,
+ _rope_parameters_are_per_layer,
+ _rope_theta_snapshot,
+ )
+
+ parameters = {
+ "full_attention": {"rope_type": "default", "rope_theta": 500000.0},
+ "sliding_attention": {"rope_type": "default", "rope_theta": 10000.0},
+ }
+ config = SimpleNamespace(rope_parameters = parameters, layer_types = ["full_attention"])
+
+ assert _rope_parameters_are_per_layer(config, parameters), (
+ "a nested rope dict was read as flat because one of its labels is absent from "
+ "layer_types; transformers asks isdisjoint, not issubset"
+ )
+ carried = _rope_theta_snapshot(config)
+ assert carried == {
+ "full_attention": 500000.0,
+ "sliding_attention": 10000.0,
+ }, f"the per-label bases were not snapshotted: {carried!r}"
+
+ config.rope_parameters = {
+ "full_attention": {"rope_type": "linear", "factor": 4.0},
+ "sliding_attention": {"rope_type": "default"},
+ }
+ carry(config, carried)
+
+ restored = config.rope_parameters
+ assert (
+ restored["full_attention"]["rope_theta"] == 500000.0
+ ), f"the full-attention base was lost: {restored['full_attention'].get('rope_theta')!r}"
+ assert restored["sliding_attention"]["rope_theta"] == 10000.0, (
+ f"the sliding-attention base was lost: "
+ f"{restored['sliding_attention'].get('rope_theta')!r}"
+ )
+ assert (
+ "rope_theta" not in restored
+ ), f"a top-level rope_theta was written into a NESTED rope dict: {restored!r}"
+ assert not hasattr(
+ config, "rope_theta"
+ ), "a nested-only config gained a scalar rope_theta attribute it never had"
+
+
+def test_rope_carry_on_the_real_nested_configs():
+ """The stub shapes above on the real config classes, so a transformers change is caught."""
+ pytest.importorskip("transformers")
+ import copy as _copy
+
+ from unsloth.import_fixes import fix_transformers_rope_scaling_drops_theta
+
+ fix_transformers_rope_scaling_drops_theta()
+
+ try:
+ from transformers.models.t5gemma2.configuration_t5gemma2 import T5Gemma2DecoderConfig
+ except Exception as exc:
+ pytest.skip(f"T5Gemma2DecoderConfig unavailable: {exc!r}")
+
+ config = T5Gemma2DecoderConfig()
+ before = config.to_json_string()
+ had_theta = hasattr(config, "rope_theta")
+ config.rope_scaling = _copy.deepcopy(config.rope_parameters)
+ assert not isinstance(
+ getattr(config, "rope_theta", None), dict
+ ), f"T5Gemma2 rope_theta became a mapping: {config.rope_theta!r}"
+ assert (
+ hasattr(config, "rope_theta") == had_theta
+ ), "a no-op assignment gave the config a rope_theta attribute it never had"
+ assert (
+ config.to_json_string() == before
+ ), "assigning a config its own rope_parameters back changed what it serializes"
+
+ try:
+ from transformers.models.deepseek_v4.configuration_deepseek_v4 import DeepseekV4Config
+ except Exception as exc:
+ pytest.skip(f"DeepseekV4Config unavailable: {exc!r}")
+
+ deepseek = DeepseekV4Config()
+ expected_compress = deepseek.rope_parameters["compress"]["rope_theta"]
+ deepseek.rope_scaling = {
+ "main": {"rope_type": "linear", "factor": 4.0},
+ "compress": {"rope_type": "default"},
+ }
+ assert (
+ "rope_theta" not in deepseek.rope_parameters
+ ), f"stray top-level rope_theta in a nested dict: {deepseek.rope_parameters!r}"
+ deepseek.standardize_rope_params()
+ resolved = deepseek.rope_parameters
+ assert resolved["compress"]["rope_theta"] == expected_compress, (
+ f"the compression base resolved to {resolved['compress']['rope_theta']!r}, "
+ f"expected {expected_compress!r}"
+ )
+
+
+def test_rope_carry_keeps_every_nested_base_on_every_real_config():
+ """Swept, not enumerated, so a new nested-rope model is covered the day it lands. On 5.17.0:
+ deepseek_v4 (``_rope_type_labels``) and laguna / mellum / zaya, all four broken by issubset."""
+ pytest.importorskip("transformers")
+
+ from unsloth.import_fixes import fix_transformers_rope_scaling_drops_theta
+
+ fix_transformers_rope_scaling_drops_theta()
+
+ from transformers.models.auto.configuration_auto import CONFIG_MAPPING
+
+ checked, damaged = [], {}
+ for name, config_class in sorted(CONFIG_MAPPING.items()):
+ try:
+ config = config_class()
+ except Exception:
+ continue
+ parameters = getattr(config, "rope_parameters", None)
+ if not isinstance(parameters, dict):
+ continue
+ labels = [k for k, v in parameters.items() if isinstance(v, dict)]
+ if len(labels) < 2:
+ continue
+ expected = {k: parameters[k].get("rope_theta") for k in labels}
+ if any(v is None for v in expected.values()):
+ continue
+ try:
+ config.rope_scaling = {k: {"rope_type": "default"} for k in labels}
+ except Exception:
+ continue
+ restored = config.rope_parameters
+ actual = {k: (restored.get(k) or {}).get("rope_theta") for k in labels}
+ checked.append(name)
+ if actual != expected or "rope_theta" in restored:
+ damaged[name] = {
+ "expected": expected,
+ "actual": actual,
+ "stray_top_level_rope_theta": "rope_theta" in restored,
+ }
+
+ assert checked, "no config with a nested rope dict was found to check"
+ assert not damaged, (
+ f"a per-label scaling replacement lost bases on {sorted(damaged)} "
+ f"(checked {len(checked)} nested configs): {damaged}"
+ )
+
+
+def test_rope_carry_leaves_the_zoo_gemma_local_base_alone_on_a_real_config():
+ """``unsloth_zoo/empty_model.py`` sets Gemma's LOCAL base, then replaces the scaling."""
+ pytest.importorskip("transformers")
+
+ from unsloth.import_fixes import fix_transformers_rope_scaling_drops_theta
+
+ fix_transformers_rope_scaling_drops_theta()
+
+ try:
+ from transformers import Gemma2Config
+ except Exception as exc:
+ pytest.skip(f"Gemma2Config unavailable: {exc!r}")
+
+ config = Gemma2Config(num_hidden_layers = 2)
+ config.rope_theta = 10000.0
+ config.rope_scaling = {"rope_type": "default"}
+
+ assert (
+ config.rope_theta == 10000.0
+ ), f"the carry overwrote the local rotary base with {config.rope_theta!r}"
+ parameters = config.rope_parameters
+ if isinstance(parameters, dict):
+ assert (
+ parameters.get("rope_theta") == 10000.0
+ ), f"the local base did not reach rope_parameters: {parameters!r}"
+
+
def test_rope_scaling_patch_wired_into_gpu_init():
source = Path(__file__).resolve().parent.parent / "unsloth" / "_gpu_init.py"
source = source.read_text(encoding = "utf-8")
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 0b0eee9f5f8..ca741249663 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -935,18 +935,27 @@ def _rope_scaling_setter_is_patched(owner = None):
return bool(getattr(prop.fset, _ROPE_SCALING_PATCH_FLAG, False))
-def _rope_parameters_are_per_layer(config, parameters):
- """Is this a per-layer-type rope dict rather than one global one?
+_ROPE_LABELS_UNSET = object()
- transformers' own discriminator: per-layer exactly when the config declares
- ``layer_types`` and every key is one of them. A global key written into such a dict
- makes transformers read the whole thing as flat, so this is asked first.
- """
- layer_types = getattr(config, "layer_types", None)
- if not layer_types or not isinstance(parameters, dict) or not parameters:
+
+def _rope_nesting_labels(config):
+ """Where ``rope_parameters`` nests: ``standardize_rope_params`` reads ``_rope_type_labels``,
+ falling back to ``layer_types``. Different axes -- DeepseekV4 keys rope by main/compress."""
+ labels = getattr(config, "_rope_type_labels", _ROPE_LABELS_UNSET)
+ if labels is _ROPE_LABELS_UNSET:
+ labels = getattr(config, "layer_types", None)
+ return labels
+
+
+def _rope_parameters_are_per_layer(config, parameters):
+ """Nested per-label rope dict or one global one? Asked first: a global key inside a nested
+ dict makes transformers read it flat. Its test is ``isdisjoint``, NOT ``issubset`` -- a
+ nested dict may name labels this config's ``layer_types`` omits."""
+ labels = _rope_nesting_labels(config)
+ if not labels or not isinstance(parameters, dict) or not parameters:
return False
try:
- return set(parameters.keys()).issubset(set(layer_types))
+ return not set(parameters.keys()).isdisjoint(set(labels))
except Exception:
return False
@@ -954,10 +963,7 @@ def _rope_parameters_are_per_layer(config, parameters):
def _rope_theta_snapshot(config):
"""The base(s) ``rope_parameters`` holds right now, in the shape it holds them.
- A flat dict has one base. A per-layer dict has one per layer type (5.5's
- ``T5Gemma2DecoderConfig``: 10000.0 sliding, 1000000.0 full), returned as
- ``{layer_type: base}``; the outer dict has no ``rope_theta``, which is how both
- bases used to be lost.
+ Flat -> one base; nested -> a ``{label: base}`` mapping, valid only one level down.
"""
parameters = getattr(config, "rope_parameters", None)
if not isinstance(parameters, dict):
@@ -976,7 +982,8 @@ def _carry_per_layer_rope_theta(config, parameters, carried):
A top-level ``rope_theta`` would make the dict read as flat, so the base goes back
into each nested entry. Copies throughout, never the caller's dicts in place. An
- entry naming its own base is left alone.
+ entry naming its own base is left alone. ``None`` means nothing was written -- no nested
+ snapshot, OR every entry already named its base, OR refused; never "fall through as scalar".
"""
if not isinstance(carried, dict) or not carried:
return None
@@ -1014,7 +1021,8 @@ def _carry_rope_theta_across_assignment(config, carried):
The base is written to ``rope_parameters`` where 5.x keeps it, so the answer does
not depend on ``standardize_rope_params`` running first and ``validate_rope`` and
- ``save_pretrained`` see a complete dict. A per-layer dict is left alone.
+ ``save_pretrained`` see a complete dict. A nested per-label dict is left alone: a
+ ``{label: base}`` ``carried`` is restored one level down or dropped, never made a scalar.
The config's ``rope_theta`` attribute (the 4.x slot) is written only when it is the
only slot that can hold the base, which is the non-dict replacement of #2405, or
@@ -1030,6 +1038,9 @@ def _carry_rope_theta_across_assignment(config, carried):
restored = _carry_per_layer_rope_theta(config, parameters, carried)
if restored is not None:
return restored
+ if isinstance(carried, dict):
+ # Nothing restored = no-op or refused; a scalar fall-through put the MAPPING here.
+ return None
elif isinstance(carried, dict):
# Per-layer parameters replaced by a FLAT dict. Passing the mapping through would
# write a dict where a number belongs. One base stands for it only when every
@@ -1047,6 +1058,10 @@ def _carry_rope_theta_across_assignment(config, carried):
base = carried
if base is None:
return None
+ if isinstance(base, dict):
+ # A base is a number. Also reachable from a config SAVED by the #11037 release,
+ # whose own rope_theta holds a mapping: refuse, it needs fixing at rest instead.
+ return None
readable = current is not None
if is_flat_dict and not readable: