Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep Mamba dt_proj out of 4-bit so QLoRA forward works (#15079)

Daniel Han995afcc86aa8
diff --git a/tests/test_mamba_dt_proj_bnb_skip.py b/tests/test_mamba_dt_proj_bnb_skip.py
new file mode 100644
index 00000000000..448ee87c195
--- /dev/null
+++ b/tests/test_mamba_dt_proj_bnb_skip.py
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Mamba QLoRA must keep mixer.dt_proj unquantized: transformers 5's Mamba slow path runs
+``self.dt_proj.weight @ time_step``, which fails on a packed Params4bit."""
+
+import torch
+
+
+def test_mamba_skips_dt_proj():
+ from unsloth.models.vision import _architecture_skip_modules
+ assert "dt_proj" in _architecture_skip_modules(["mamba"])
+
+
+def test_falcon_mamba_and_others_unchanged():
+ from unsloth.models.vision import _architecture_skip_modules
+
+ assert "dt_proj" not in _architecture_skip_modules(["falcon_mamba"])
+ assert "dt_proj" not in _architecture_skip_modules(["mamba2"])
+ assert _architecture_skip_modules(["llama"]) == []
+
+
+def test_user_bnb_config_gets_mamba_dt_proj():
+ from transformers import BitsAndBytesConfig
+ from unsloth.models.vision import _with_architecture_skip_modules
+
+ user = BitsAndBytesConfig(load_in_4bit = True, llm_int8_skip_modules = ["lm_head"])
+ out = _with_architecture_skip_modules(user, ["mamba"])
+ assert out.llm_int8_skip_modules == ["lm_head", "dt_proj"]
+ assert user.llm_int8_skip_modules == ["lm_head"]
+
+
+def test_skip_key_keeps_tiny_mamba_dt_proj_out_of_bnb():
+ import pytest
+
+ pytest.importorskip("bitsandbytes")
+ from transformers import MambaConfig, MambaForCausalLM
+ from transformers.integrations import bitsandbytes as tbnb
+ from unsloth.models.vision import _architecture_skip_modules
+ from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES
+
+ config = MambaConfig(vocab_size = 64, hidden_size = 16, state_size = 4, num_hidden_layers = 2, expand = 2)
+ with torch.device("meta"):
+ model = MambaForCausalLM(config)
+ skip = list(SKIP_QUANTIZATION_MODULES) + _architecture_skip_modules(["mamba"])
+ from transformers import BitsAndBytesConfig
+
+ qc = BitsAndBytesConfig(load_in_4bit = True, llm_int8_skip_modules = skip)
+ try:
+ model = tbnb.replace_with_bnb_linear(
+ model, modules_to_not_convert = skip, quantization_config = qc
+ )
+ except TypeError:
+ model = tbnb.replace_with_bnb_linear(
+ model, modules_to_not_convert = skip, quantization_config = qc, pre_quantized = False
+ )
+ import bitsandbytes as bnb
+
+ kinds = {
+ name: type(m).__name__
+ for name, m in model.named_modules()
+ if name.endswith(("dt_proj", "in_proj", "x_proj"))
+ }
+ assert all(
+ not isinstance(model.get_submodule(n), bnb.nn.Linear4bit)
+ for n in kinds
+ if n.endswith("dt_proj")
+ ), kinds
+ assert any(
+ isinstance(model.get_submodule(n), bnb.nn.Linear4bit)
+ for n in kinds
+ if n.endswith("in_proj")
+ ), kinds
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 6b7db80ca02..6aafd48ffa6 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -2730,6 +2730,9 @@ def _architecture_skip_modules(model_types):
# Nemotron-H uses 'mixer' (not 'mamba') for Mamba layers, whose fused kernels pass out_proj.weight straight to F.linear and fail with quantized Params4bit, so skip out_proj.
if any(mt == "nemotron_h" for mt in model_types):
skip.append("out_proj")
+ # Mamba's transformers 5 slow path does dt_proj.weight @ time_step, which fails on Params4bit; dt_proj is tiny.
+ if any(mt == "mamba" for mt in model_types):
+ skip.append("dt_proj")
# LongCat-Flash MLA: NF4 on q_b_proj / kv_b_proj badly hurts loss; both are small.
if any(mt in ("longcat_flash", "longcat_flash_lsa") for mt in model_types):
skip.extend(("q_b_proj", "kv_b_proj"))