Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep LoRA adapters on Mamba x_proj / dt_proj / out_proj when the fused training kernel is available (#15539)
Daniel Han986558ebf9af
diff --git a/tests/test_mamba_fused_lora_adapters.py b/tests/test_mamba_fused_lora_adapters.pynew file mode 100644index 00000000000..c4b86645566--- /dev/null+++ b/tests/test_mamba_fused_lora_adapters.py@@ -0,0 +1,475 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""LoRA on a Mamba projection the fused training kernel reads raw (x_proj / dt_proj / out_proj) must train.++transformers' fused training branch passes `self.x_proj.weight` (and dt_proj / out_proj) to+`mamba_inner_fn`, so an adapter on those modules is bypassed. On CPU the fused kernel is simulated by+a stub that, like the real kernel, only sees the weights it is handed: plain LoRA must keep the fused+call and hand it `W + delta`; other adapters must route the mixer to its non-fused path.+"""++import ast+import importlib+import importlib.util+import inspect+import pathlib+import threading+import warnings++import pytest+import torch+import torch.nn.functional as F++peft = pytest.importorskip("peft")++ARCHS = {+ "mamba": ("MambaConfig", "MambaForCausalLM", "MambaMixer"),+ "falcon_mamba": ("FalconMambaConfig", "FalconMambaForCausalLM", "FalconMambaMixer"),+}+# out_proj is read raw too, but PEFT refuses LoRA on it for these model types.+RAW = ("x_proj", "dt_proj")+++_ROOT = pathlib.Path(__file__).resolve().parents[1]+_FIX = _ROOT / "unsloth" / "models" / "_mamba_lora.py"+++def _patch_fn():+ # Loaded by path (torch + peft only); a no-op on trees without the fix.+ if not _FIX.is_file():+ return lambda model: model+ spec = importlib.util.spec_from_file_location("_unsloth_mamba_lora_under_test", _FIX)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module.patch_mamba_fused_lora+++def _fake_fused(+ xz, conv1d_weight, conv1d_bias, x_proj_weight, delta_proj_weight, out_proj_weight, *a, **k+):+ out = F.linear(xz.chunk(2, dim = 1)[0].transpose(1, 2), out_proj_weight)+ return out + 1e-3 * (x_proj_weight.sum() + delta_proj_weight.sum())+++def _reference_fused(+ xz,+ conv1d_weight,+ conv1d_bias,+ x_proj_weight,+ delta_proj_weight,+ out_proj_weight,+ out_proj_bias,+ A,+ B = None,+ C = None,+ D = None,+ delta_bias = None,+ delta_softplus = True,+ **kwargs,+):+ # mamba_ssm's mamba_inner_fn math (mamba_inner_ref) on the weights it is handed.+ seq_len = xz.shape[-1]+ x, z = xz.chunk(2, dim = 1)+ x = F.conv1d(+ x,+ conv1d_weight,+ conv1d_bias,+ padding = conv1d_weight.shape[-1] - 1,+ groups = x.shape[1],+ )+ x = F.silu(x[..., :seq_len])+ rank, n = delta_proj_weight.shape[1], A.shape[1]+ dt, Bm, Cm = torch.split(F.linear(x.transpose(1, 2), x_proj_weight), [rank, n, n], dim = -1)+ delta = delta_proj_weight @ dt.transpose(1, 2)+ if delta_bias is not None:+ delta = delta + delta_bias[None, :, None]+ if delta_softplus:+ delta = F.softplus(delta)+ state = xz.new_zeros(x.shape[0], x.shape[1], n)+ ys = []+ for t in range(seq_len):+ state = (+ torch.exp(delta[:, :, t, None] * A) * state+ + delta[:, :, t, None] * Bm[:, t, None, :] * x[:, :, t, None]+ )+ ys.append((state * Cm[:, t, None, :]).sum(-1))+ y = (torch.stack(ys, dim = -1) + x * D[None, :, None]) * F.silu(z)+ return F.linear(y.transpose(1, 2), out_proj_weight, out_proj_bias)+++def _build(+ monkeypatch,+ arch,+ targets,+ kernel = _fake_fused,+ lora_dropout = 0.0,+ fused = True,+ patch = True,+ forward_kwargs = False,+):+ import transformers++ config_name, model_name, mixer_name = ARCHS[arch]+ modeling = importlib.import_module(f"transformers.models.{arch}.modeling_{arch}")+ mixer_cls = getattr(modeling, mixer_name)+ calls = []++ # Both transformers lines call the module-level mamba_inner_fn in training without a cache;+ # transformers 5 treats a None return as "no kernel", 4.x returns its result directly.+ def mamba_inner_fn(+ xz, conv1d_weight, conv1d_bias, x_proj_weight, delta_proj_weight, out_proj_weight, *a, **k+ ):+ calls.append(+ {+ "x_proj": x_proj_weight,+ "dt_proj": delta_proj_weight,+ "out_proj": out_proj_weight,+ }+ )+ if not fused:+ return None+ return kernel(+ xz,+ conv1d_weight,+ conv1d_bias,+ x_proj_weight,+ delta_proj_weight,+ out_proj_weight,+ *a,+ **k,+ )++ monkeypatch.setattr(modeling, "mamba_inner_fn", mamba_inner_fn)+ monkeypatch.setattr(mixer_cls, "forward", mixer_cls.forward) # undo what the fix installs+ if hasattr(mixer_cls, "cuda_kernels_forward"):+ # transformers 4.x dispatches to cuda_kernels_forward only on CUDA with mamba_ssm; force it,+ # or its torch path for the non-fused reference.+ def forward(+ self,+ hidden_states,+ cache_params = None,+ cache_position = None,+ attention_mask = None,+ ):+ impl = self.cuda_kernels_forward if fused else self.slow_forward+ return impl(hidden_states, cache_params, cache_position, attention_mask)++ if forward_kwargs:+ # transformers 5.5 / 5.10: the dispatching forward also takes **kwargs, slow_forward does not.+ def forward(+ self,+ hidden_states,+ cache_params = None,+ cache_position = None,+ attention_mask = None,+ **kwargs,+ ):+ impl = self.cuda_kernels_forward if fused else self.slow_forward+ return impl(hidden_states, cache_params, cache_position, attention_mask)++ monkeypatch.setattr(mixer_cls, "forward", forward)++ config = getattr(transformers, config_name)(+ vocab_size = 64,+ hidden_size = 16,+ state_size = 4,+ num_hidden_layers = 2,+ intermediate_size = 32,+ time_step_rank = 4,+ conv_kernel = 4,+ use_cache = False,+ )+ torch.manual_seed(0)+ model = getattr(transformers, model_name)(config)+ if patch:+ _patch_fn()(model) # what FastModel.from_pretrained does at load+ model = peft.get_peft_model(+ model,+ peft.LoraConfig(r = 4, lora_alpha = 8, lora_dropout = lora_dropout, target_modules = targets),+ )+ if patch:+ _patch_fn()(model) # and get_peft_model afterwards+ model.train()+ torch.manual_seed(1)+ with torch.no_grad():+ for name, p in model.named_parameters():+ if ".lora_B." in name:+ p.normal_(std = 0.3)+ return model, calls+++def _loss(model, ids):+ return model(input_ids = ids, labels = ids, use_cache = False).loss+++def _lora_param(model, leaf, kind):+ return [p for n, p in model.named_parameters() if f".{leaf}.lora_{kind}." in n]+++def _mixers(model):+ return [layer.mixer for layer in model.base_model.model.backbone.layers]+++def _legacy(arch):+ modeling = importlib.import_module(f"transformers.models.{arch}.modeling_{arch}")+ return hasattr(getattr(modeling, ARCHS[arch][2]), "slow_forward")++[email protected]("arch", list(ARCHS))+def test_plain_lora_keeps_fused_kernel_with_merged_weights(monkeypatch, arch):+ model, calls = _build(monkeypatch, arch, ["in_proj", *RAW])+ ids = torch.randint(0, 64, (2, 8))++ loss = _loss(model, ids)+ assert len(calls) == 2, "fused kernel must stay on for plain LoRA"+ for mixer, seen in zip(_mixers(model), calls):+ for leaf in RAW:+ module = mixer._modules[leaf]+ expected = module.get_base_layer().weight + module.get_delta_weight("default")+ assert seen[leaf] is not module.get_base_layer().weight, leaf+ torch.testing.assert_close(seen[leaf], expected)+ loss.backward()+ for leaf in ("in_proj", *RAW):+ for kind in ("A", "B"):+ grads = [p.grad for p in _lora_param(model, leaf, kind)]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads), (leaf, kind)+ for name, p in model.named_parameters():+ if not p.requires_grad:+ assert p.grad is None, name++ for leaf in RAW:+ with torch.no_grad():+ before = model(input_ids = ids, use_cache = False).logits+ for p in _lora_param(model, leaf, "B"):+ p.mul_(2.0)+ after = model(input_ids = ids, use_cache = False).logits+ assert (after - before).abs().max() > 1e-5, leaf++[email protected]("arch", list(ARCHS))[email protected]("dropout", [0.0, 0.1])+def test_dispatching_forward_with_trailing_kwargs_is_patched(monkeypatch, arch, dropout):+ # transformers 5.5 / 5.10: forward(..., **kwargs) over slow_forward / cuda_kernels_forward without it.+ if not _legacy(arch):+ pytest.skip("this transformers has no slow_forward / cuda_kernels_forward split")+ model, calls = _build(+ monkeypatch, arch, ["in_proj", "x_proj"], lora_dropout = dropout, forward_kwargs = True+ )+ ids = torch.randint(0, 64, (2, 8))+ _loss(model, ids).backward()+ assert len(calls) == (0 if dropout else 2)+ for mixer in _mixers(model):+ assert mixer(torch.randn(2, 8, 16), unused_by_slow_forward = 1).shape == (2, 8, 16)+ for kind in ("A", "B"):+ grads = [p.grad for p in _lora_param(model, "x_proj", kind)]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads), kind+++def _assert_fused_with_merged_x_proj(model, calls, ids):+ calls.clear()+ _loss(model, ids).backward()+ assert len(calls) == 2, "fused kernel must stay on for plain LoRA"+ for mixer, seen in zip(_mixers(model), calls):+ assert seen["x_proj"] is not mixer.x_proj.get_base_layer().weight+ grads = [p.grad for p in _lora_param(model, "x_proj", "B")]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)++[email protected]("arch", list(ARCHS))+def test_kernel_global_reassigned_after_patching_is_gated_again(monkeypatch, arch):+ # transformers 5.5 / 5.10 MambaMixer.__init__ reassigns the module's mamba_inner_fn, e.g. when a+ # second model is built after the first was patched.+ model, calls = _build(monkeypatch, arch, ["in_proj", "x_proj"])+ modeling = importlib.import_module(f"transformers.models.{arch}.modeling_{arch}")+ modeling.mamba_inner_fn = inspect.unwrap(modeling.mamba_inner_fn)+ _assert_fused_with_merged_x_proj(model, calls, torch.randint(0, 64, (2, 8)))++[email protected]("arch", list(ARCHS))+def test_routing_state_is_per_thread(monkeypatch, arch):+ # nn.DataParallel replicas run concurrently: another thread's frame or force-slow must not leak in.+ model, calls = _build(monkeypatch, arch, ["in_proj", "x_proj"])+ routing = type(_mixers(model)[0]).forward.__globals__["_routing"]+ routing.force_slow += 1+ routing.stack.append(None)+ errors = []++ def run():+ try:+ _assert_fused_with_merged_x_proj(model, calls, torch.randint(0, 64, (2, 8)))+ except BaseException as e:+ errors.append(e)++ try:+ thread = threading.Thread(target = run)+ thread.start()+ thread.join(120)+ finally:+ routing.force_slow -= 1+ routing.stack.pop()+ assert not thread.is_alive()+ if errors:+ raise errors[0]+++def test_merged_weight_grads_match_non_fused_path(monkeypatch):+ # transformers 5's torch path reads dt_proj.weight raw too, so it is only a reference on 4.x.+ targets = ["in_proj", "x_proj"] + (["dt_proj"] if _legacy("mamba") else [])+ ids = torch.randint(0, 64, (2, 8), generator = torch.Generator().manual_seed(3))+ results = {}+ for fused in (True, False):+ model, calls = _build(+ monkeypatch,+ "mamba",+ targets,+ kernel = _reference_fused,+ fused = fused,+ patch = fused,+ )+ loss = _loss(model, ids)+ loss.backward()+ assert len(calls) == (2 if fused else (0 if _legacy("mamba") else 2))+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if ".lora_" in n}+ results[fused] = (loss.detach(), grads)+ monkeypatch.undo()+ (loss_f, grads_f), (loss_ref, grads_ref) = results[True], results[False]+ torch.testing.assert_close(loss_f, loss_ref, rtol = 1e-5, atol = 1e-6)+ assert grads_f.keys() == grads_ref.keys()+ for leaf in targets:+ names = [n for n in grads_ref if f".{leaf}.lora_" in n]+ assert names, leaf+ for n in names:+ assert grads_ref[n].abs().sum() > 0, n+ torch.testing.assert_close(grads_f[n], grads_ref[n], rtol = 1e-4, atol = 1e-6)++[email protected]("arch", list(ARCHS))+def test_dropout_routes_to_non_fused_path(monkeypatch, arch):+ model, calls = _build(monkeypatch, arch, ["in_proj", "x_proj"], lora_dropout = 0.1)+ ids = torch.randint(0, 64, (2, 8))+ _loss(model, ids).backward()+ assert calls == [], "dropout on x_proj cannot be merged into the weight"+ for kind in ("A", "B"):+ grads = [p.grad for p in _lora_param(model, "x_proj", kind)]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads), kind+++def _slow_calls(arch, leaf):+ modeling = importlib.import_module(f"transformers.models.{arch}.modeling_{arch}")+ mixer_cls = getattr(modeling, ARCHS[arch][2])+ slow = mixer_cls.slow_forward if _legacy(arch) else mixer_cls.forward+ return f"self.{leaf}(" in inspect.getsource(inspect.unwrap(slow))++[email protected]("arch", list(ARCHS))+def test_dt_proj_lora_trains_on_fused_kernel(monkeypatch, arch):+ model, calls = _build(monkeypatch, arch, ["in_proj", "dt_proj"])+ ids = torch.randint(0, 64, (2, 8))+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ loss = _loss(model, ids)+ assert not any("not applied" in str(w.message) for w in caught)+ assert len(calls) == 2+ for mixer, seen in zip(_mixers(model), calls):+ expected = mixer.dt_proj.get_base_layer().weight + mixer.dt_proj.get_delta_weight("default")+ torch.testing.assert_close(seen["dt_proj"], expected)+ assert seen["x_proj"] is mixer.x_proj.weight+ loss.backward()+ for kind in ("A", "B"):+ grads = [p.grad for p in _lora_param(model, "dt_proj", kind)]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads), kind++[email protected]("arch", list(ARCHS))+def test_unmergeable_dt_proj_routes_only_where_the_slow_path_calls_it(monkeypatch, arch):+ # transformers 5's MambaMixer torch path computes dt_proj.weight @ time_step, so leaving the+ # fused kernel for a dt_proj adapter with dropout would cost speed and still train nothing:+ # stay fused and warn. 4.x's slow_forward calls self.dt_proj(...), so there it routes and trains.+ model, calls = _build(monkeypatch, arch, ["in_proj", "dt_proj"], lora_dropout = 0.1)+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")+ _loss(model, torch.randint(0, 64, (2, 8))).backward()+ warned = any("dt_proj" in str(w.message) for w in caught)+ grads = [p.grad for p in _lora_param(model, "dt_proj", "B")]+ trained = bool(grads) and all(g is not None and g.abs().sum() > 0 for g in grads)+ if _slow_calls(arch, "dt_proj"):+ assert calls == []+ else:+ assert len(calls) == 2, "left the fused kernel for an adapter the slow path ignores"+ if _legacy(arch):+ assert trained and not warned+ assert trained != warned, (trained, warned)+++def test_dt_proj_lora_without_kernel_on_transformers_5_warns(monkeypatch):+ # No fused kernel (the entry point returns None): transformers 5 falls through to its torch path,+ # which reads dt_proj.weight raw, so even plain LoRA there cannot train.+ if _legacy("mamba"):+ pytest.skip("transformers 4.x slow_forward calls dt_proj as a module")+ model, calls = _build(monkeypatch, "mamba", ["in_proj", "x_proj", "dt_proj"], fused = False)+ with pytest.warns(UserWarning, match = r"MambaMixer\.dt_proj") as caught:+ _loss(model, torch.randint(0, 64, (2, 8))).backward()+ assert not any("x_proj" in str(w.message) for w in caught)+ assert len(calls) == 2+ grads = [p.grad for p in _lora_param(model, "x_proj", "B")]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)++[email protected]("arch", list(ARCHS))+def test_multiple_active_adapters_route_to_non_fused_path(monkeypatch, arch):+ model, calls = _build(monkeypatch, arch, ["in_proj", "x_proj"])+ model.add_adapter(+ "second", peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["in_proj", "x_proj"])+ )+ model.base_model.set_adapter(["default", "second"])+ model.train()+ _loss(model, torch.randint(0, 64, (2, 8))).backward()+ assert calls == []+ grads = [p.grad for p in _lora_param(model, "x_proj", "A")]+ assert grads and all(g is not None for g in grads)++[email protected]("arch", list(ARCHS))+def test_fused_path_kept_without_raw_weight_adapters(monkeypatch, arch):+ model, calls = _build(monkeypatch, arch, ["in_proj"])+ ids = torch.randint(0, 64, (2, 8))+ _loss(model, ids).backward()+ assert len(calls) == 2 # one per layer+ for mixer, seen in zip(_mixers(model), calls):+ for leaf in RAW:+ assert seen[leaf] is mixer._modules[leaf].weight, leaf+ grads = [p.grad for p in _lora_param(model, "in_proj", "B")]+ assert grads and all(g is not None for g in grads)++[email protected]("arch", list(ARCHS))+def test_disabled_adapter_keeps_raw_weights(monkeypatch, arch):+ model, calls = _build(monkeypatch, arch, ["in_proj", "x_proj"])+ ids = torch.randint(0, 64, (2, 8))+ with model.disable_adapter():+ _loss(model, ids)+ assert len(calls) == 2+ for mixer, seen in zip(_mixers(model), calls):+ assert seen["x_proj"] is mixer.x_proj.get_base_layer().weight+ calls.clear()+ _loss(model, ids)+ assert len(calls) == 2+ for mixer, seen in zip(_mixers(model), calls):+ assert seen["x_proj"] is not mixer.x_proj.get_base_layer().weight+++def test_post_patch_model_installs_the_routing():+ # FastModel.from_pretrained and get_peft_model both end in FastBaseModel.post_patch_model.+ tree = ast.parse((_ROOT / "unsloth" / "models" / "vision.py").read_text())+ post = next(+ node+ for node in ast.walk(tree)+ if isinstance(node, ast.FunctionDef) and node.name == "post_patch_model"+ )+ called = {+ node.func.id+ for node in ast.walk(post)+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)+ }+ assert "patch_mamba_fused_lora" in calleddiff --git a/unsloth/models/_mamba_lora.py b/unsloth/models/_mamba_lora.pynew file mode 100644index 00000000000..6c15c5eb8aa--- /dev/null+++ b/unsloth/models/_mamba_lora.py@@ -0,0 +1,330 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Keep LoRA adapters on Mamba projections that the fused training kernels read as raw weights.++In training (no cache) transformers' Mamba mixers hand ``self.x_proj.weight``, ``self.dt_proj.weight``+and ``self.out_proj.weight`` (Mamba2-style mixers: ``self.out_proj.weight``) straight to+``mamba_inner_fn`` / ``mamba_split_conv1d_scan_combined``, so a PEFT adapter wrapped around those+modules never runs and gets no gradient.++When every such adapter is plain LoRA (one active adapter, no dropout, no DoRA / variant, no LoRA+bias, dense ``nn.Linear`` base) the fused kernel is kept and receives ``W + delta`` instead of ``W``,+with ``delta`` from PEFT's own ``get_delta_weight`` built inside autograd, so lora_A / lora_B get+exact gradients at fused-kernel memory and speed. Otherwise the call routes to the mixer's own+non-fused path when that path calls more of the adapted modules (transformers 5's Mamba torch path+still reads dt_proj.weight raw, so it does not help a dt_proj adapter). An adapter neither path+applies warns once. Mixers without raw-weight adapters are unchanged.+"""++import functools+import inspect+import sys+import threading+import warnings++import torch++__all__ = ["patch_mamba_fused_lora"]++# Mixer leaves a fused kernel reads as `.weight` instead of calling the module.+_RAW_WEIGHT_LEAVES = ("x_proj", "dt_proj", "out_proj")+# Fused entry points; transformers 5 falls back to its torch path when one returns None.+_FUSED_FNS = (+ "mamba_inner_fn",+ "mamba_split_conv1d_scan_combined",+ "mamba2_split_conv1d_scan_combined",+)+_MARK = "_unsloth_mamba_fused_lora"+++class _Routing(threading.local):+ # Per thread: nn.DataParallel replicas run their forwards concurrently.+ def __init__(self):+ self.force_slow = 0+ self.stack = []+++_routing = _Routing()+_warned = set()+++class _Frame:+ __slots__ = ("merged", "applied")++ def __init__(self, merged):+ # {id(base weight): (base weight, W + delta, leaf)} handed to the fused kernel.+ self.merged = merged+ # Leaves whose adapter ran: merged into a kernel call that returned, or called as a module.+ self.applied = set()+++def _has_active_adapter(module):+ try:+ from peft.tuners.tuners_utils import BaseTunerLayer+ except Exception:+ return False+ if not isinstance(module, BaseTunerLayer):+ return False+ # Merged adapters already live in base_layer.weight, which is what the kernel reads.+ if module.disable_adapters or module.merged:+ return False+ for layer_name in module.adapter_layer_names:+ layer = getattr(module, layer_name, None)+ if layer is not None and any(name in layer for name in module.active_adapters):+ return True+ return False+++def mamba_raw_weight_adapters(mixer):+ """Names of the mixer's raw-weight leaves that carry an active, unmerged adapter."""+ return [name for name in _RAW_WEIGHT_LEAVES if _has_active_adapter(mixer._modules.get(name))]+++def _merged_weight(module):+ """``W + delta`` when that is exactly what the LoRA module computes, else None."""+ try:+ from peft.tuners.lora import Linear as LoraLinear+ except Exception:+ return None+ if not isinstance(module, LoraLinear):+ return None+ active = [name for name in module.active_adapters if name in module.lora_A]+ if len(active) != 1:+ return None+ # Other adapter layers (embedding LoRA, a mixed tuner) for the active adapter are not covered.+ for layer_name in module.adapter_layer_names:+ if layer_name not in ("lora_A", "lora_B"):+ layer = getattr(module, layer_name, None)+ if layer is not None and active[0] in layer:+ return None+ name = active[0]+ if name in getattr(module, "lora_variant", {}) or getattr(module, "use_dora", {}).get(name):+ return None+ if getattr(module, "fan_in_fan_out", False):+ return None+ if getattr(module.lora_B[name], "bias", None) is not None:+ return None+ dropout = module.lora_dropout[name]+ if not isinstance(dropout, (torch.nn.Identity, torch.nn.Dropout)):+ return None+ if isinstance(dropout, torch.nn.Dropout) and dropout.training and dropout.p > 0:+ return None+ base = module.get_base_layer()+ # Exact nn.Linear only: bitsandbytes / other quantized subclasses do not hold a dense weight.+ if type(base) is not torch.nn.Linear:+ return None+ weight = base.weight+ if not weight.is_floating_point() or weight.is_meta:+ return None+ return weight + module.get_delta_weight(name).to(weight.dtype)+++def _swap(value, merged, used):+ if isinstance(value, torch.Tensor):+ entry = merged.get(id(value))+ if entry is not None and entry[0] is value:+ used.append(entry[2])+ return entry[1]+ return value+++def _gate(fn):+ if getattr(fn, _MARK, False):+ return fn++ @functools.wraps(fn)+ def gated(*args, **kwargs):+ if _routing.force_slow:+ return None+ stack = _routing.stack+ if not stack or not stack[-1].merged:+ return fn(*args, **kwargs)+ frame, used = stack[-1], []+ args = tuple(_swap(value, frame.merged, used) for value in args)+ kwargs = {key: _swap(value, frame.merged, used) for key, value in kwargs.items()}+ out = fn(*args, **kwargs)+ if out is not None:+ frame.applied.update(used)+ return out++ setattr(gated, _MARK, True)+ return gated+++def _is_mamba_mixer(module):+ name = type(module).__name__+ if not (name.endswith("Mixer") or hasattr(module, "ssm_state_size")):+ return False+ return any(module._modules.get(leaf) is not None for leaf in _RAW_WEIGHT_LEAVES)+++def _calls_fused(fn):+ code = getattr(inspect.unwrap(fn), "__code__", None)+ return code is not None and any(name in code.co_names for name in _FUSED_FNS)+++def _leaves_called(fn):+ """Raw-weight leaves the non-fused path calls as modules, so an adapter on them runs there.++ transformers 5's MambaMixer torch path computes ``self.dt_proj.weight @ time_step``, so routing a+ dt_proj adapter there would trade the fused kernel for nothing. Unknown source: assume all.+ """+ try:+ source = inspect.getsource(inspect.unwrap(fn))+ except (OSError, TypeError):+ return frozenset(_RAW_WEIGHT_LEAVES)+ return frozenset(leaf for leaf in _RAW_WEIGHT_LEAVES if f"self.{leaf}(" in source)+++def _mark_applied(applied, leaf, module, args):+ applied.add(leaf)+++def _warn_not_applied(cls, leaves):+ leaves = [leaf for leaf in leaves if (cls, leaf) not in _warned]+ if not leaves:+ return+ _warned.update((cls, leaf) for leaf in leaves)+ names = ", ".join(f"{cls.__name__}.{leaf}" for leaf in leaves)+ warnings.warn(+ f"Unsloth: the LoRA adapter on {names} is not applied in this Mamba training forward and "+ "will not train. The fused kernel reads that weight raw and only plain LoRA (one active "+ "adapter, lora_dropout = 0, no DoRA, no LoRA bias, dense base) can be merged into it, and "+ "the non-fused path does not call it either. Use lora_dropout = 0 or remove it from "+ "target_modules.",+ stacklevel = 3,+ )+++def _named_params(fn):+ return [+ name+ for name, p in inspect.signature(fn).parameters.items()+ if p.kind not in (p.VAR_POSITIONAL, p.VAR_KEYWORD)+ ]+++def _keywords_taken(fn):+ """Keyword names ``fn`` accepts, or None when it takes **kwargs."""+ params = inspect.signature(fn).parameters.values()+ if any(p.kind is p.VAR_KEYWORD for p in params):+ return None+ return {p.name for p in params if p.kind is not p.VAR_POSITIONAL}+++def _gate_module_fns(cls):+ # Both lines look the fused function up in the modeling module's globals at call time, and+ # transformers 5.5 / 5.10 MambaMixer.__init__ reassigns them: re-gate on every use.+ module_globals = getattr(sys.modules.get(cls.__module__), "__dict__", {})+ names = [name for name in _FUSED_FNS if callable(module_globals.get(name))]+ for name in names:+ module_globals[name] = _gate(module_globals[name])+ return names+++def _patch_class(cls):+ forward = cls.forward+ if getattr(forward, _MARK, False):+ _gate_module_fns(cls)+ return True+ params = list(inspect.signature(forward).parameters)+ if "cache_params" not in params or len(params) < 2:+ return False+ # Position of cache_params within *args, after (self, hidden_states).+ cache_index = params.index("cache_params") - 2++ # transformers 4.x: forward picks cuda_kernels_forward or a same-signature torch path.+ slow_takes = None+ slow = getattr(cls, "slow_forward", None) or getattr(cls, "torch_forward", None)+ cuda_forward = getattr(cls, "cuda_kernels_forward", None)+ if callable(cuda_forward) and callable(slow):+ if not _calls_fused(cuda_forward):+ return False+ # transformers 5.5 / 5.10 forward adds **kwargs that it drops before calling slow_forward.+ if _named_params(slow)[2:] != _named_params(forward)[2:]:+ return False+ slow_takes = _keywords_taken(slow)+ legacy = True+ else:+ # transformers 5: one forward whose fused branch short-circuits unless the kernel returns None.+ if not _calls_fused(forward):+ return False+ legacy = False+ if not _gate_module_fns(cls) and not legacy:+ return False+ called = _leaves_called(slow if legacy else forward)++ def non_fused(self, hidden_states, *args, **kwargs):+ if legacy:+ if slow_takes is not None:+ kwargs = {k: v for k, v in kwargs.items() if k in slow_takes}+ return slow(self, hidden_states, *args, **kwargs)+ _routing.force_slow += 1+ try:+ return forward(self, hidden_states, *args, **kwargs)+ finally:+ _routing.force_slow -= 1++ @functools.wraps(forward)+ def forward_keep_adapters(self, hidden_states, *args, **kwargs):+ if self.training:+ if "cache_params" in kwargs:+ cache_params = kwargs["cache_params"]+ else:+ cache_params = args[cache_index] if 0 <= cache_index < len(args) else None+ leaves = mamba_raw_weight_adapters(self) if cache_params is None else None+ if leaves:+ names = _gate_module_fns(cls)+ modules = {leaf: self._modules[leaf] for leaf in leaves}+ weights = {leaf: _merged_weight(modules[leaf]) for leaf in leaves} if names else {}+ fused_cover = [leaf for leaf in leaves if weights.get(leaf) is not None]+ # Leave the fused kernel only when the non-fused path applies more adapters.+ use_slow = sum(leaf in called for leaf in leaves) > len(fused_cover)+ merged = {}+ if not use_slow:+ for leaf in fused_cover:+ base = modules[leaf].get_base_layer().weight+ merged[id(base)] = (base, weights[leaf], leaf)+ frame = _Frame(merged)+ hooks = [+ module.register_forward_pre_hook(+ functools.partial(_mark_applied, frame.applied, leaf)+ )+ for leaf, module in modules.items()+ ]+ _routing.stack.append(frame)+ try:+ if use_slow:+ out = non_fused(self, hidden_states, *args, **kwargs)+ else:+ # Fused kernel kept; it receives W + delta in place of each adapted W.+ out = forward(self, hidden_states, *args, **kwargs)+ finally:+ _routing.stack.pop()+ for hook in hooks:+ hook.remove()+ _warn_not_applied(cls, [leaf for leaf in leaves if leaf not in frame.applied])+ return out+ return forward(self, hidden_states, *args, **kwargs)++ setattr(forward_keep_adapters, _MARK, True)+ cls.forward = forward_keep_adapters+ return True+++def patch_mamba_fused_lora(model):+ """Keep LoRA on Mamba mixers whose fused kernels read x_proj / dt_proj / out_proj raw."""+ if not isinstance(model, torch.nn.Module):+ return model+ seen = set()+ for module in model.modules():+ cls = type(module)+ if cls in seen or not _is_mamba_mixer(module):+ continue+ seen.add(cls)+ try:+ _patch_class(cls)+ except Exception:+ pass+ return modeldiff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 01e82297b41..4277cd36913 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -282,6 +282,7 @@ from ._uma_safetensors import is_integrated_unified_memory_gpufrom ._utils import estimate_training_reserve_bytes as _zoo_reserve_estimatefrom ._remote_code_buffers import restore_remote_code_non_persistent_buffersfrom ._custom_dtype import cast_mixed_linear_inputs, resolve_dtype, trusted_custom_dtype+from ._mamba_lora import patch_mamba_fused_lorafrom .remote_code_shims import apply_remote_code_shimsfrom .grouped_linear_lora import register_grouped_linear_lorafrom .loader_utils import (@@ -5458,6 +5459,8 @@ class FastBaseModel:):reject_alora(model)set_deepspeed_moe_leaf_modules(model)+ # Mamba fused kernels read x_proj / dt_proj / out_proj .weight raw, skipping adapters on them.+ patch_mamba_fused_lora(model)from ..import_fixes import fix_transformers_voxtral_realtime_inplace_audio_addfix_transformers_voxtral_realtime_inplace_audio_add()