Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Mamba LoRA: apply a trained dt_proj / x_proj / out_proj adapter in eval and cached decode (#15730)

Daniel Han802c5898c291
diff --git a/tests/test_mamba_dt_proj_eval_parity.py b/tests/test_mamba_dt_proj_eval_parity.py
new file mode 100644
index 00000000000..4d11ecbf714
--- /dev/null
+++ b/tests/test_mamba_dt_proj_eval_parity.py
@@ -0,0 +1,676 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A trained LoRA on Mamba's dt_proj must also apply in eval, cached decode and generate.
+
+transformers' Mamba torch path (transformers 5 always, 4.x ``cuda_kernels_forward`` outside fused
+training) computes ``self.dt_proj.weight @ time_step``, which reads the base weight and skips the
+adapter. Unmerged eval / generate must equal the ``merge_and_unload`` model, ``disable_adapter()``
+must give the base model, and the fused training path must stay compile friendly.
+"""
+
+import copy
+import importlib
+import importlib.util
+import pathlib
+import pickle
+import subprocess
+import sys
+import threading
+import warnings
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+peft = pytest.importorskip("peft")
+
+_HERE = pathlib.Path(__file__).resolve().parent
+_spec = importlib.util.spec_from_file_location(
+ "_mamba_fused_lora_helpers", _HERE / "test_mamba_fused_lora_adapters.py"
+)
+helpers = importlib.util.module_from_spec(_spec)
+sys.modules[_spec.name] = helpers # torch.compile resolves the fake kernel's globals by module
+_spec.loader.exec_module(helpers)
+
+ARCHS = list(helpers.ARCHS)
+ALL = ["in_proj", "x_proj", "dt_proj"]
+
+
+# transformers 4.x: the test models always dispatch to cuda_kernels_forward, which outside fused
+# training reads dt_proj.weight raw. Its mamba_ssm / causal_conv1d ops get their reference math here
+# so prefill and cached decode run on CPU.
+def _causal_conv1d_ref(
+ x,
+ weight,
+ bias = None,
+ activation = None,
+ **kwargs,
+):
+ seq_len = x.shape[-1]
+ out = F.conv1d(x, weight.unsqueeze(1), bias, padding = weight.shape[-1] - 1, groups = x.shape[1])
+ out = out[..., :seq_len]
+ return F.silu(out) if activation in ("silu", "swish") else out
+
+
+def _selective_scan_ref(
+ u,
+ delta,
+ A,
+ B,
+ C,
+ D = None,
+ z = None,
+ delta_bias = None,
+ delta_softplus = False,
+ return_last_state = False,
+):
+ delta = delta.float()
+ if delta_bias is not None:
+ delta = delta + delta_bias[None, :, None].float()
+ if delta_softplus:
+ delta = F.softplus(delta)
+ state = u.new_zeros(u.shape[0], u.shape[1], A.shape[1], dtype = torch.float32)
+ ys = []
+ for t in range(u.shape[-1]):
+ state = (
+ torch.exp(delta[:, :, t, None] * A) * state
+ + delta[:, :, t, None] * B[:, None, :, t] * u[:, :, t, None]
+ )
+ ys.append((state * C[:, None, :, t]).sum(-1))
+ y = torch.stack(ys, dim = -1)
+ if D is not None:
+ y = y + u * D[None, :, None]
+ if z is not None:
+ y = y * F.silu(z)
+ y = y.to(u.dtype)
+ return (y, state) if return_last_state else y
+
+
+def _causal_conv1d_update_ref(
+ x,
+ conv_state,
+ weight,
+ bias = None,
+ activation = None,
+ **kwargs,
+):
+ conv_state.copy_(torch.roll(conv_state, shifts = -1, dims = -1))
+ conv_state[:, :, -1] = x
+ out = (conv_state * weight).sum(-1)
+ if bias is not None:
+ out = out + bias
+ return F.silu(out) if activation in ("silu", "swish") else out
+
+
+def _selective_state_update_ref(
+ state,
+ x,
+ dt,
+ A,
+ B,
+ C,
+ D = None,
+ z = None,
+ dt_bias = None,
+ dt_softplus = False,
+ **kwargs,
+):
+ dt = dt.float()
+ if dt_bias is not None:
+ dt = dt + dt_bias.float()
+ if dt_softplus:
+ dt = F.softplus(dt)
+ new = torch.exp(dt[..., None] * A) * state + dt[..., None] * B[:, None, :] * x[..., None]
+ state.copy_(new)
+ y = (new * C[:, None, :]).sum(-1)
+ if D is not None:
+ y = y + x * D
+ if z is not None:
+ y = y * F.silu(z)
+ return y.to(x.dtype)
+
+
+def _install_legacy_reference_ops(monkeypatch, arch):
+ modeling = importlib.import_module(f"transformers.models.{arch}.modeling_{arch}")
+ ops = {
+ "selective_scan_fn": _selective_scan_ref,
+ "selective_state_update": _selective_state_update_ref,
+ "_lazy_load_causal_conv1d": lambda: (_causal_conv1d_update_ref, _causal_conv1d_ref),
+ # transformers < 4.56 binds the causal_conv1d ops as module globals instead of lazily.
+ "causal_conv1d_fn": _causal_conv1d_ref,
+ "causal_conv1d_update": _causal_conv1d_update_ref,
+ }
+ for name, fn in ops.items():
+ monkeypatch.setattr(modeling, name, fn, raising = False)
+
+
+def _trained(monkeypatch, arch, targets, **kwargs):
+ """A model whose adapters took one step through the fused training path, then put in eval."""
+ model, calls = helpers._build(
+ monkeypatch, arch, targets, kernel = helpers._reference_fused, **kwargs
+ )
+ ids = torch.randint(0, 64, (2, 8), generator = torch.Generator().manual_seed(5))
+ helpers._loss(model, ids).backward()
+ assert calls, "training did not take the fused path"
+ grads = [p.grad for p in helpers._lora_param(model, "dt_proj", "B")]
+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if p.grad is not None:
+ p -= 10.0 * p.grad
+ model.zero_grad(set_to_none = True)
+ model.eval()
+ if helpers._legacy(arch):
+ _install_legacy_reference_ops(monkeypatch, arch)
+ return model
+
+
+def _merged(model):
+ return copy.deepcopy(model).merge_and_unload().eval()
+
+
+def _logits(model, ids):
+ with torch.no_grad():
+ return model(input_ids = ids, use_cache = False).logits
+
+
+def _ids(seed = 7):
+ return torch.randint(0, 64, (2, 12), generator = torch.Generator().manual_seed(seed))
+
+
[email protected]("arch", ARCHS)
[email protected]("targets", [["dt_proj"], ALL], ids = ["dt_proj", "all"])
+def test_eval_logits_match_merged(monkeypatch, arch, targets):
+ model = _trained(monkeypatch, arch, targets)
+ ids = _ids()
+ unmerged, merged = _logits(model, ids), _logits(_merged(model), ids)
+ with model.disable_adapter():
+ base = _logits(model, ids)
+ assert (merged - base).abs().max() > 1e-6, "the trained adapter has no effect"
+ if targets == ["dt_proj"]:
+ # dt_proj is only read as a weight, so W + delta is the merged weight bit for bit.
+ assert torch.equal(unmerged, merged), (unmerged - merged).abs().max()
+ else:
+ torch.testing.assert_close(unmerged, merged, rtol = 1e-5, atol = 1e-5)
+
+
[email protected]("arch", ARCHS)
+def test_greedy_generate_matches_merged(monkeypatch, arch):
+ model = _trained(monkeypatch, arch, ALL)
+ merged = _merged(model)
+ ids = _ids(11)[:1, :6]
+ kwargs = dict(
+ max_new_tokens = 8,
+ do_sample = False,
+ use_cache = True,
+ output_scores = True,
+ return_dict_in_generate = True,
+ pad_token_id = 0,
+ )
+ with torch.no_grad():
+ out_u = model.generate(input_ids = ids, **kwargs)
+ out_m = merged.generate(input_ids = ids, **kwargs)
+ assert torch.equal(out_u.sequences, out_m.sequences)
+ for su, sm in zip(out_u.scores, out_m.scores):
+ torch.testing.assert_close(su, sm, rtol = 1e-5, atol = 1e-5)
+
+
[email protected]("arch", ARCHS)
+def test_disable_adapter_gives_base_model(monkeypatch, arch):
+ model = _trained(monkeypatch, arch, ALL)
+ ids = _ids()
+ base = _logits(copy.deepcopy(model).unload().eval(), ids)
+ with model.disable_adapter():
+ assert torch.equal(_logits(model, ids), base)
+ assert not torch.equal(_logits(model, ids), base)
+
+
[email protected]("arch", ARCHS)
+def test_eval_weight_follows_updates_and_adapter_switches(monkeypatch, arch):
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids()
+ first = _logits(model, ids)
+ # An in-place optimizer-style update must show in the next eval forward.
+ with torch.no_grad():
+ for p in helpers._lora_param(model, "dt_proj", "B"):
+ p.mul_(-1.5)
+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))
+ assert not torch.equal(_logits(model, ids), first)
+
+ model.add_adapter("second", peft.LoraConfig(r = 2, lora_alpha = 4, target_modules = ["dt_proj"]))
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if ".lora_B.second." in name:
+ p.normal_(std = 0.5)
+ model.eval()
+ for active in ("second", "default"):
+ model.set_adapter(active)
+ model.eval()
+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids)), active
+ model.base_model.set_adapter(["default", "second"])
+ model.eval()
+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))
+
+ # merge_adapter() folds the delta into the base weight: no double counting, and unmerge restores.
+ unmerged = _logits(model, ids)
+ model.merge_adapter()
+ torch.testing.assert_close(_logits(model, ids), unmerged, rtol = 1e-5, atol = 1e-6)
+ model.unmerge_adapter()
+ torch.testing.assert_close(_logits(model, ids), unmerged, rtol = 1e-5, atol = 1e-6)
+
+
[email protected]("arch", ARCHS)
+def test_eval_weight_follows_data_reassignment(monkeypatch, arch):
+ # `p.data = t` (custom LoRA inits, dtype casts at save time) keeps id(p) and p._version.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids()
+ first = _logits(model, ids)
+ for p in helpers._lora_param(model, "dt_proj", "B"):
+ p.data = p.data * -1.5
+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))
+ assert not torch.equal(_logits(model, ids), first)
+
+
[email protected]("arch", ARCHS)
+def test_eval_weight_follows_dtype_conversion(monkeypatch, arch):
+ # model.double() / .bfloat16() keep the parameter objects and their version counters.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids()
+ _logits(model, ids)
+ model.double()
+ assert all(p.dtype == torch.float64 for p in helpers._lora_param(model, "dt_proj", "B"))
+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))
+
+
[email protected]("arch", ARCHS)
+def test_eval_weight_follows_unversioned_optimizer_step(monkeypatch, arch):
+ # FlashAdamW's Triton update writes the parameters in place without bumping their version
+ # counters (as an in-place op through `p.data` does), so eval -> train step -> eval must not
+ # serve the W + delta of the first eval.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids()
+ first = _logits(model, ids)
+ model.train()
+ helpers._loss(model, ids).backward()
+ for p in helpers._lora_param(model, "dt_proj", "B"):
+ version = p._version
+ p.data.add_(p.grad.data, alpha = -10.0)
+ p.data.mul_(-1.5)
+ assert p._version == version
+ model.zero_grad(set_to_none = True)
+ model.eval()
+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))
+ assert not torch.equal(_logits(model, ids), first)
+
+
[email protected]("arch", ARCHS)
+def test_eval_holds_no_effective_weight_copy(monkeypatch, arch):
+ # W + delta is rebuilt per read, never stored on the leaf, so model.cpu() / .to() leave no
+ # stale copy behind and every later read sees the converted weights.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids()
+ _logits(model, ids)
+ leaves = [m for m in model.modules() if type(m).__name__ == "Linear" and hasattr(m, "lora_A")]
+ assert leaves
+ for leaf in leaves:
+ assert not any(isinstance(v, torch.Tensor) for v in leaf.__dict__.values())
+ assert not any(
+ isinstance(v, tuple) and any(isinstance(t, torch.Tensor) for t in v)
+ for v in leaf.__dict__.values()
+ )
+ model.to(torch.float64)
+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))
+
+
[email protected]("arch", ARCHS)
+def test_autocast_eval_does_not_leak_into_full_precision(monkeypatch, arch):
+ # W + delta is built outside autocast: no bf16-rounded delta from an earlier autocast forward.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids()
+ with torch.autocast("cpu", dtype = torch.bfloat16):
+ _logits(model, ids)
+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))
+
+
[email protected]("arch", ARCHS)
+def test_mixed_adapter_batch_routes_each_row(monkeypatch, arch):
+ # PEFT's adapter_names= picks an adapter per row ("__base__" = none) through LoRA pre-hooks that a
+ # raw weight read bypasses: each row must still get exactly its own adapter.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids()
+ adapted = _logits(model, ids)
+ with model.disable_adapter():
+ base = _logits(model, ids)
+ assert (adapted - base).abs().max() > 1e-6, "the trained adapter has no effect"
+ with torch.no_grad():
+ mixed = model(input_ids = ids, use_cache = False, adapter_names = ["__base__", "default"]).logits
+ torch.testing.assert_close(mixed[0], base[0], rtol = 1e-6, atol = 1e-6)
+ torch.testing.assert_close(mixed[1], adapted[1], rtol = 1e-6, atol = 1e-6)
+ # The mixed batch must not poison the plain path afterwards.
+ assert torch.equal(_logits(model, ids), adapted)
+
+
+def test_concurrent_eval_threads_each_see_the_adapter(monkeypatch):
+ # nn.DataParallel replicas and serving threads run forwards concurrently: the effective weight
+ # is armed per thread, so one thread finishing cannot disarm another mid-forward.
+ model = _trained(monkeypatch, "mamba", ["dt_proj"])
+ ids = _ids()
+ expected = _logits(_merged(model), ids)
+ results, errors = [], []
+
+ def run():
+ try:
+ for _ in range(5):
+ results.append(torch.equal(_logits(model, ids), expected))
+ except BaseException as e:
+ errors.append(e)
+
+ threads = [threading.Thread(target = run) for _ in range(4)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(120)
+ if errors:
+ raise errors[0]
+ assert len(results) == 20 and all(results)
+
+
+def test_pickled_model_loads_without_unsloth(monkeypatch, tmp_path):
+ # torch.save(model) / pickle / deepcopy must stay loadable in a process that never ran the patch.
+ model = _trained(monkeypatch, "mamba", ["dt_proj"])
+ ids = _ids()
+ expected = _logits(model, ids)
+ leaf = model.base_model.model.backbone.layers[0].mixer.dt_proj
+ data = pickle.dumps(model)
+ assert b"_RawRead_" not in data
+ # copy / deepcopy keep the effective-weight class (see the compiled deepcopy test below).
+ assert type(copy.deepcopy(leaf)) is type(leaf)
+ assert torch.equal(_logits(pickle.loads(data), ids), expected)
+ path = tmp_path / "model.pkl"
+ path.write_bytes(data)
+ code = (
+ "import pickle, sys; m = pickle.load(open(sys.argv[1], 'rb')); "
+ "print(type(m.base_model.model.backbone.layers[0].mixer.dt_proj).__qualname__)"
+ )
+ out = subprocess.run([sys.executable, "-c", code, str(path)], capture_output = True, text = True)
+ assert out.returncode == 0, out.stderr[-2000:]
+ assert out.stdout.strip() == type(leaf).__mro__[1].__qualname__
+
+
+def test_unfoldable_adapter_warns_on_raw_read(monkeypatch):
+ if helpers._legacy("mamba"):
+ pytest.skip("transformers 4.x eval on CPU calls dt_proj as a module")
+ model, _ = helpers._build(monkeypatch, "mamba", ["dt_proj"], fused = False)
+ model = peft.get_peft_model(
+ model.unload(),
+ peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["dt_proj"], use_dora = True),
+ )
+ helpers._patch_fn()(model)
+ model.eval()
+ with pytest.warns(UserWarning, match = r"MambaMixer\.dt_proj is not applied"):
+ _logits(model, _ids())
+
+
[email protected]("arch", ARCHS)
+def test_fused_training_mixer_compiles_fullgraph(monkeypatch, arch):
+ model, calls = helpers._build(monkeypatch, arch, ALL, kernel = helpers._reference_fused)
+ mixer = helpers._mixers(model)[0]
+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))
+ params = [p for n, p in mixer.named_parameters() if ".lora_" in n]
+
+ def run(fn):
+ for p in params:
+ p.grad = None
+ out = fn(hidden)
+ out.float().pow(2).sum().backward()
+ return out.detach(), [p.grad.clone() for p in params]
+
+ out_e, grads_e = run(mixer)
+ torch._dynamo.reset()
+ compiled = torch.compile(mixer, backend = "aot_eager", fullgraph = True)
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore")
+ out_c, grads_c = run(compiled)
+ run(compiled)
+ torch._dynamo.reset()
+ torch.testing.assert_close(out_c, out_e, rtol = 1e-5, atol = 1e-6)
+ for gc, ge in zip(grads_c, grads_e):
+ assert ge.abs().sum() > 0
+ torch.testing.assert_close(gc, ge, rtol = 1e-4, atol = 1e-6)
+
+
+def _hotswap_fn():
+ try:
+ from peft.utils.hotswap import hotswap_adapter
+ except Exception:
+ return None
+ return hotswap_adapter
+
+
[email protected]("arch", ARCHS)
+def test_hotswapped_adapter_applies_in_eval(monkeypatch, arch, tmp_path):
+ # PEFT's hotswap_adapter replaces same-shaped LoRA weights through `old.data.copy_(new)`, which
+ # keeps each parameter's identity, storage and version counter; eval must serve the new adapter.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids()
+ other = copy.deepcopy(model)
+ with torch.no_grad():
+ for name, p in other.named_parameters():
+ if ".lora_" in name:
+ p.normal_(std = 2.0, generator = torch.Generator().manual_seed(3))
+ expected = _logits(_merged(other), ids)
+ old = _logits(model, ids) # an eval forward before the swap
+ assert (expected - old).abs().max() > 1e-3, "the replacement adapter has no effect"
+ hotswap = _hotswap_fn()
+ if hotswap is not None:
+ other.save_pretrained(tmp_path)
+ hotswap(model, str(tmp_path), adapter_name = "default", torch_device = "cpu")
+ else:
+ new = dict(other.named_parameters())
+ for name, p in model.named_parameters():
+ if ".lora_" in name:
+ p.data.copy_(new[name].data)
+ # No model.eval() / train() here: a hot swap in a serving process calls neither.
+ assert torch.equal(_logits(model, ids), expected)
+
+
[email protected]("arch", ARCHS)
+def test_eval_mixer_compiles_fullgraph_under_no_grad(monkeypatch, arch):
+ # A no-grad eval forward reads dt_proj.weight raw; under torch.compile it must trace in one
+ # graph (no cache lookups or mutations inside it) and match eager.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if ".lora_" in name:
+ p.normal_(std = 2.0, generator = torch.Generator().manual_seed(3))
+ mixer = helpers._mixers(model)[0]
+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))
+ with torch.no_grad():
+ eager = mixer(hidden)
+ with model.disable_adapter():
+ base = mixer(hidden)
+ assert (eager - base).abs().max() > 1e-5, "the adapter has no effect"
+ torch._dynamo.reset()
+ try:
+ compiled = torch.compile(mixer, backend = "aot_eager", fullgraph = True)
+ with warnings.catch_warnings(), torch.no_grad():
+ warnings.simplefilter("ignore")
+ out = compiled(hidden)
+ again = compiled(hidden)
+ finally:
+ torch._dynamo.reset()
+ torch.testing.assert_close(out, eager, rtol = 1e-5, atol = 1e-6)
+ assert torch.equal(out, again)
+
+
+def _adapted_mixer(monkeypatch, arch):
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if ".lora_" in name:
+ p.normal_(std = 2.0, generator = torch.Generator().manual_seed(3))
+ return model, helpers._mixers(model)[0]
+
+
+def _compiled_no_grad(
+ mixer,
+ hidden,
+ fullgraph = True,
+):
+ torch._dynamo.reset()
+ try:
+ compiled = torch.compile(mixer, backend = "aot_eager", fullgraph = fullgraph)
+ with warnings.catch_warnings(), torch.no_grad():
+ warnings.simplefilter("ignore")
+ out = compiled(hidden)
+ again = compiled(hidden)
+ finally:
+ torch._dynamo.reset()
+ assert torch.equal(out, again)
+ return out
+
+
+def _eager_and_base(model, mixer, hidden):
+ with torch.no_grad():
+ eager = mixer(hidden)
+ with model.disable_adapter():
+ base = mixer(hidden)
+ assert (eager - base).abs().max() > 1e-5, "the adapter has no effect"
+ return eager
+
+
[email protected]("arch", ARCHS)
+def test_deepcopied_eval_mixer_compiles_with_its_adapter(monkeypatch, arch):
+ # A deep copy compiled before any eager forward must still serve W + delta, not the base weight.
+ model, mixer = _adapted_mixer(monkeypatch, arch)
+ copied = copy.deepcopy(mixer)
+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))
+ out = _compiled_no_grad(copied, hidden)
+ torch.testing.assert_close(out, _eager_and_base(model, mixer, hidden), rtol = 1e-5, atol = 1e-6)
+ assert type(copied.dt_proj) is type(mixer.dt_proj)
+
+
[email protected]("arch", ARCHS)
+def test_adapter_attached_after_patch_compiles_with_it(monkeypatch, arch):
+ # A dt_proj adapter added after the patch, compiled before any eager forward, must apply.
+ model, _ = helpers._build(monkeypatch, arch, ["x_proj"], kernel = helpers._reference_fused)
+ if helpers._legacy(arch):
+ _install_legacy_reference_ops(monkeypatch, arch)
+ model.add_adapter("late", peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["dt_proj"]))
+ model.set_adapter("late")
+ model.eval()
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if ".lora_" in name and ".late." in name:
+ p.normal_(std = 2.0, generator = torch.Generator().manual_seed(3))
+ mixer = helpers._mixers(model)[0]
+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))
+ out = _compiled_no_grad(mixer, hidden)
+ torch.testing.assert_close(out, _eager_and_base(model, mixer, hidden), rtol = 1e-5, atol = 1e-6)
+
+
[email protected]("arch", ARCHS)
+def test_unprepared_adapter_never_compiles_silently(monkeypatch, arch):
+ # A leaf that lost its effective-weight class: fullgraph fails loudly, otherwise the frame runs
+ # eagerly (which installs it) and every call applies the adapter.
+ model, mixer = _adapted_mixer(monkeypatch, arch)
+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))
+ eager = _eager_and_base(model, mixer, hidden)
+ plain = type(mixer.dt_proj).__mro__[1]
+ mixer.dt_proj.__class__ = plain
+ with pytest.raises(Exception, match = r"dt_proj has an active LoRA adapter"):
+ _compiled_no_grad(mixer, hidden)
+ mixer.dt_proj.__class__ = plain
+ out = _compiled_no_grad(mixer, hidden, fullgraph = False)
+ torch.testing.assert_close(out, eager, rtol = 1e-5, atol = 1e-6)
+
+
+def _row_logits(model, sequences, prompt_len, names):
+ """Per-step logits of each generated row, recomputed with that row's adapter alone."""
+ out = []
+ for row, name in zip(sequences, names):
+ if name == "__base__":
+ with model.disable_adapter():
+ logits = _logits(model, row[None])
+ else:
+ logits = _logits(model, row[None])
+ out.append(logits[0, prompt_len - 1 : -1])
+ return torch.stack(out)
+
+
[email protected]("arch", ARCHS)
+ "names", [["__base__", "default"], ["default", "default"]], ids = ["mixed", "same"]
+)
[email protected]("greedy", [False, True], ids = ["sample", "greedy"])
+def test_mixed_adapter_generate_num_return_sequences(monkeypatch, arch, names, greedy):
+ # generate expands each prompt num_return_sequences times (repeat_interleave), while PEFT's
+ # adapter_names hook keeps one name per prompt: every expanded row keeps its prompt's adapter.
+ # transformers refuses num_return_sequences without sampling, so greedy is top_k = 1 sampling.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids(13)[:, :4]
+ torch.manual_seed(0)
+ with torch.no_grad():
+ out = model.generate(
+ input_ids = ids,
+ adapter_names = names,
+ do_sample = True,
+ top_k = 1 if greedy else 0,
+ num_return_sequences = 2,
+ max_new_tokens = 3,
+ use_cache = True,
+ output_logits = True,
+ return_dict_in_generate = True,
+ pad_token_id = 0,
+ )
+ assert out.sequences.shape[0] == 4
+ row_names = [name for name in names for _ in range(2)]
+ assert torch.equal(out.sequences[:, :4], ids.repeat_interleave(2, dim = 0))
+ got = torch.stack(out.logits, dim = 1)
+ expected = _row_logits(model, out.sequences, 4, row_names)
+ torch.testing.assert_close(got, expected, rtol = 1e-5, atol = 1e-5)
+ if greedy:
+ for i, name in enumerate(names):
+ kwargs = dict(max_new_tokens = 3, do_sample = False, use_cache = True, pad_token_id = 0)
+ with torch.no_grad():
+ if name == "__base__":
+ with model.disable_adapter():
+ alone = model.generate(input_ids = ids[i : i + 1], **kwargs)
+ else:
+ alone = model.generate(input_ids = ids[i : i + 1], **kwargs)
+ assert torch.equal(out.sequences[2 * i], alone[0])
+ assert torch.equal(out.sequences[2 * i + 1], alone[0])
+
+
[email protected]("arch", ARCHS)
+def test_mixed_adapter_generate_beam_search(monkeypatch, arch):
+ # PEFT repeats adapter_names num_beams times itself; they must be used as given.
+ model = _trained(monkeypatch, arch, ["dt_proj"])
+ ids = _ids(17)[:, :4]
+ names = ["__base__", "default"]
+ kwargs = dict(
+ max_new_tokens = 3,
+ num_beams = 2,
+ num_return_sequences = 2,
+ do_sample = False,
+ use_cache = True,
+ pad_token_id = 0,
+ )
+ with torch.no_grad():
+ out = model.generate(input_ids = ids, adapter_names = names, **kwargs)
+ assert out.shape[0] == 4
+ for i, name in enumerate(names):
+ if name == "__base__":
+ with model.disable_adapter():
+ alone = model.generate(input_ids = ids[i : i + 1], **kwargs)
+ else:
+ alone = model.generate(input_ids = ids[i : i + 1], **kwargs)
+ assert torch.equal(out[2 * i : 2 * i + 2], alone)
+
+
+def test_mixed_adapter_names_not_dividing_batch_raise(monkeypatch):
+ model = _trained(monkeypatch, "mamba", ["dt_proj"])
+ ids = torch.randint(0, 64, (3, 6), generator = torch.Generator().manual_seed(3))
+ with torch.no_grad(), pytest.raises(ValueError, match = "adapter_names"):
+ model(input_ids = ids, use_cache = False, adapter_names = ["__base__", "default"])
diff --git a/tests/test_mamba_fused_lora_adapters.py b/tests/test_mamba_fused_lora_adapters.py
index 504a6459ef5..822eb2b655c 100644
--- a/tests/test_mamba_fused_lora_adapters.py
+++ b/tests/test_mamba_fused_lora_adapters.py
@@ -14,6 +14,7 @@ import importlib
import importlib.util
import inspect
import pathlib
+import sys
import threading
import warnings
@@ -36,12 +37,17 @@ _FIX = _ROOT / "unsloth" / "models" / "_mamba_lora.py"
def _patch_fn():
- # Loaded by path (torch + peft only); a no-op on trees without the fix.
+ # Loaded by path (torch + peft only), once, like a real import; a no-op on trees without the fix.
if not _FIX.is_file():
return lambda model: model
- spec = importlib.util.spec_from_file_location("_unsloth_mamba_lora_under_test", _FIX)
- module = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(module)
+ name = "_unsloth_mamba_lora_under_test"
+ module = sys.modules.get(name)
+ if module is None:
+ spec = importlib.util.spec_from_file_location(name, _FIX)
+ module = importlib.util.module_from_spec(spec)
+ # torch.compile resolves classes through sys.modules.
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
return module.patch_mamba_fused_lora
@@ -142,6 +148,7 @@ def _build(
monkeypatch.setattr(modeling, "mamba_inner_fn", mamba_inner_fn)
monkeypatch.setattr(mixer_cls, "forward", mixer_cls.forward) # undo what the fix installs
+ monkeypatch.setattr(mixer_cls, "__setattr__", mixer_cls.__setattr__)
if hasattr(mixer_cls, "cuda_kernels_forward"):
# transformers 4.x dispatches to cuda_kernels_forward only on CUDA with mamba_ssm; force it,
# or its torch path for the non-fused reference.
@@ -402,18 +409,20 @@ def test_unmergeable_dt_proj_routes_only_where_the_slow_path_calls_it(monkeypatc
assert trained != warned, (trained, warned)
-def test_dt_proj_lora_without_kernel_on_transformers_5_warns(monkeypatch):
+def test_dt_proj_lora_without_kernel_on_transformers_5_trains(monkeypatch):
# No fused kernel (the entry point returns None): transformers 5 falls through to its torch path,
- # which reads dt_proj.weight raw, so even plain LoRA there cannot train.
+ # which reads dt_proj.weight raw; the effective-weight property makes that read W + delta.
if _legacy("mamba"):
pytest.skip("transformers 4.x slow_forward calls dt_proj as a module")
model, calls = _build(monkeypatch, "mamba", ["in_proj", "x_proj", "dt_proj"], fused = False)
- with pytest.warns(UserWarning, match = r"MambaMixer\.dt_proj") as caught:
+ with warnings.catch_warnings(record = True) as caught:
+ warnings.simplefilter("always")
_loss(model, torch.randint(0, 64, (2, 8))).backward()
- assert not any("x_proj" in str(w.message) for w in caught)
+ assert not any("not applied" in str(w.message) for w in caught)
assert len(calls) == 2
- grads = [p.grad for p in _lora_param(model, "x_proj", "B")]
- assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)
+ for leaf in ("x_proj", "dt_proj"):
+ grads = [p.grad for p in _lora_param(model, leaf, "B")]
+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads), leaf
@pytest.mark.parametrize("arch", list(ARCHS))
diff --git a/unsloth/models/_mamba_lora.py b/unsloth/models/_mamba_lora.py
index 6c15c5eb8aa..7d28805bef3 100644
--- a/unsloth/models/_mamba_lora.py
+++ b/unsloth/models/_mamba_lora.py
@@ -12,11 +12,18 @@ When every such adapter is plain LoRA (one active adapter, no dropout, no DoRA /
bias, dense ``nn.Linear`` base) the fused kernel is kept and receives ``W + delta`` instead of ``W``,
with ``delta`` from PEFT's own ``get_delta_weight`` built inside autograd, so lora_A / lora_B get
exact gradients at fused-kernel memory and speed. Otherwise the call routes to the mixer's own
-non-fused path when that path calls more of the adapted modules (transformers 5's Mamba torch path
-still reads dt_proj.weight raw, so it does not help a dt_proj adapter). An adapter neither path
-applies warns once. Mixers without raw-weight adapters are unchanged.
+non-fused path when that path calls more of the adapted modules. An adapter neither path applies
+warns once. Mixers without raw-weight adapters are unchanged.
+
+The torch path (transformers 5 always, 4.x ``cuda_kernels_forward`` outside fused training) also
+reads ``self.dt_proj.weight @ time_step`` raw, in eval and cached decode too. Adapted leaves therefore
+get a ``weight`` property that, while their mixer's forward runs, returns ``W + delta`` (built inside
+autograd and outside autocast, rebuilt on every read so no weight write can leave it stale), so
+unmerged eval and generate match ``merge_and_unload`` and ``disable_adapter()`` gives the base model.
"""
+import copy
+import copyreg
import functools
import inspect
import sys
@@ -43,17 +50,22 @@ class _Routing(threading.local):
def __init__(self):
self.force_slow = 0
self.stack = []
+ # [(adapted leaf, W + delta or None = build on read, "Mixer.leaf", batch size or None)]
+ self.raw = []
_routing = _Routing()
_warned = set()
+_TUNER_LAYER = None
+_RAW_READ_CLASSES = {}
class _Frame:
__slots__ = ("merged", "applied")
def __init__(self, merged):
- # {id(base weight): (base weight, W + delta, leaf)} handed to the fused kernel.
+ # [(weight the forward passes, W + delta handed to the kernel instead, leaf)]; matched by
+ # identity, not id(), which torch.compile cannot trace for a tensor built in the forward.
self.merged = merged
# Leaves whose adapter ran: merged into a kernel call that returned, or called as a module.
self.applied = set()
@@ -76,40 +88,62 @@ def _has_active_adapter(module):
return False
+def _tuner_layer_class():
+ global _TUNER_LAYER
+ if _TUNER_LAYER is None:
+ try:
+ from peft.tuners.tuners_utils import BaseTunerLayer
+ except Exception:
+ return ()
+ _TUNER_LAYER = BaseTunerLayer
+ return _TUNER_LAYER
+
+
def mamba_raw_weight_adapters(mixer):
"""Names of the mixer's raw-weight leaves that carry an active, unmerged adapter."""
- return [name for name in _RAW_WEIGHT_LEAVES if _has_active_adapter(mixer._modules.get(name))]
+ modules, tuner = mixer._modules, _TUNER_LAYER or _tuner_layer_class()
+ leaves = [name for name in _RAW_WEIGHT_LEAVES if isinstance(modules.get(name), tuner)]
+ return [name for name in leaves if _has_active_adapter(modules[name])] if leaves else leaves
-def _merged_weight(module):
- """``W + delta`` when that is exactly what the LoRA module computes, else None."""
+def _merged_weight(
+ module,
+ allow_multiple = False,
+ names = None,
+):
+ """``W + delta`` when that is exactly what the LoRA module computes, else None.
+
+ ``names`` replaces the active adapters (one row of a PEFT mixed-adapter batch).
+ """
try:
from peft.tuners.lora import Linear as LoraLinear
except Exception:
return None
if not isinstance(module, LoraLinear):
return None
- active = [name for name in module.active_adapters if name in module.lora_A]
- if len(active) != 1:
- return None
- # Other adapter layers (embedding LoRA, a mixed tuner) for the active adapter are not covered.
- for layer_name in module.adapter_layer_names:
- if layer_name not in ("lora_A", "lora_B"):
- layer = getattr(module, layer_name, None)
- if layer is not None and active[0] in layer:
- return None
- name = active[0]
- if name in getattr(module, "lora_variant", {}) or getattr(module, "use_dora", {}).get(name):
+ if names is None:
+ names = module.active_adapters
+ active = [name for name in names if name in module.lora_A]
+ if not active or (len(active) != 1 and not allow_multiple):
return None
+ for name in active:
+ # Other adapter layers (embedding LoRA, a mixed tuner) for the active adapter are not covered.
+ for layer_name in module.adapter_layer_names:
+ if layer_name not in ("lora_A", "lora_B"):
+ layer = getattr(module, layer_name, None)
+ if layer is not None and name in layer:
+ return None
+ if name in getattr(module, "lora_variant", {}) or getattr(module, "use_dora", {}).get(name):
+ return None
+ if getattr(module.lora_B[name], "bias", None) is not None:
+ return None
+ dropout = module.lora_dropout[name]
+ if not isinstance(dropout, (torch.nn.Identity, torch.nn.Dropout)):
+ return None
+ if isinstance(dropout, torch.nn.Dropout) and dropout.training and dropout.p > 0:
+ return None
if getattr(module, "fan_in_fan_out", False):
return None
- if getattr(module.lora_B[name], "bias", None) is not None:
- return None
- dropout = module.lora_dropout[name]
- if not isinstance(dropout, (torch.nn.Identity, torch.nn.Dropout)):
- return None
- if isinstance(dropout, torch.nn.Dropout) and dropout.training and dropout.p > 0:
- return None
base = module.get_base_layer()
# Exact nn.Linear only: bitsandbytes / other quantized subclasses do not hold a dense weight.
if type(base) is not torch.nn.Linear:
@@ -117,15 +151,195 @@ def _merged_weight(module):
weight = base.weight
if not weight.is_floating_point() or weight.is_meta:
return None
- return weight + module.get_delta_weight(name).to(weight.dtype)
+ # Same order as PEFT's merge (base += delta per adapter), so merged and unmerged match bitwise.
+ for name in active:
+ weight = weight + module.get_delta_weight(name).to(weight.dtype)
+ return weight
+
+
+def _full_precision_merged(module, names = None):
+ # Built outside autocast, as merge_and_unload does, so autocast eval matches the merged model.
+ device_type = module.get_base_layer().weight.device.type
+ try:
+ # Autocast off already: same tensor without the context's cost, which shows on decode reads.
+ context = None
+ if torch.compiler.is_compiling() or torch.is_autocast_enabled(device_type):
+ context = torch.autocast(device_type, enabled = False)
+ except Exception:
+ context = None
+ if context is None:
+ return _merged_weight(module, allow_multiple = True, names = names)
+ with context:
+ return _merged_weight(module, allow_multiple = True, names = names)
+
+
+def _mixed_batch_adapter_names(module):
+ """Per-row adapter names of a PEFT mixed-adapter batch (``adapter_names=``), else None.
+
+ PEFT delivers them through a forward pre-hook on each LoRA layer, which a raw ``.weight``
+ read never runs.
+ """
+ for hook in module._forward_pre_hooks.values():
+ if (
+ isinstance(hook, functools.partial)
+ and getattr(hook.func, "__name__", "") == "_adapter_names_pre_forward_hook"
+ ):
+ names = hook.keywords.get("adapter_names")
+ if names is not None:
+ return list(names)
+ return None
+
+
+def _per_row_weight(
+ module,
+ names,
+ label,
+ batch = None,
+):
+ """[batch, out, in] effective weights, one per row of a mixed-adapter batch.
+
+ ``W @ x`` with a [batch, in, seq] input broadcasts per row, so each row sees its own adapter and
+ ``"__base__"`` / unknown names see the base weight, as PEFT's mixed-batch forward gives them.
+
+ PEFT repeats the names only for ``num_beams`` (peft/tuners/lora/model.py
+ ``_enable_peft_forward_hooks``); ``generate`` also expands each prompt ``num_return_sequences``
+ times with ``repeat_interleave``, so a running batch that is a multiple of the names gets them
+ repeated in that same order. Any other size cannot be routed and raises.
+ """
+ if batch is not None and batch != len(names):
+ if not names or batch % len(names):
+ raise ValueError(
+ f"Unsloth: {label} got {len(names)} adapter_names for a batch of {batch} rows; "
+ "pass one adapter name per input row."
+ )
+ repeat = batch // len(names)
+ names = [name for name in names for _ in range(repeat)]
+ base = module.get_base_layer().weight
+ weights = {}
+ for name in dict.fromkeys(names):
+ if name == "__base__" or name not in module.lora_A:
+ weights[name] = base
+ continue
+ weight = _full_precision_merged(module, names = [name])
+ if weight is None:
+ if not torch.compiler.is_compiling():
+ _warn_raw_read(label)
+ weight = base
+ weights[name] = weight
+ return torch.stack([weights[name] for name in names])
+
+
+def _warn_raw_read(label):
+ if ("raw_read", label) in _warned:
+ return
+ _warned.add(("raw_read", label))
+ warnings.warn(
+ f"Unsloth: the adapter on {label} is not applied here: this Mamba forward reads that "
+ "weight raw and only plain LoRA (no DoRA, no LoRA bias, no active dropout, dense base) can "
+ "be folded into it. Merge the adapter (merge_and_unload) for inference.",
+ stacklevel = 3,
+ )
+
+
+def _raw_read_class(cls):
+ """Subclass of an adapter layer whose ``weight`` is ``W + delta`` while its mixer forward runs."""
+ sub = _RAW_READ_CLASSES.get(cls)
+ if sub is not None:
+ return sub
+
+ def weight(self):
+ # Per thread: nn.DataParallel replicas share their leaves' __dict__ contents.
+ for module, effective, label, batch in reversed(_routing.raw):
+ if module is not self:
+ continue
+ if effective is None:
+ names = _mixed_batch_adapter_names(self)
+ if names is not None:
+ return _per_row_weight(self, names, label, batch)
+ # Never cached: hotswap_adapter / FlashAdamW `.data` writes skip version counters.
+ effective = _full_precision_merged(self)
+ if effective is not None:
+ return effective
+ if not torch.compiler.is_compiling():
+ _warn_raw_read(label)
+ break
+ return super(sub, self).weight
+
+ def __reduce_ex__(self, protocol):
+ # Pickle as the plain adapter layer so it loads without Unsloth.
+ return copyreg._reconstructor, (cls, object, None), self.__dict__.copy()
+
+ # copy / deepcopy keep this class: a trace cannot install it and would read the base weight.
+ def __copy__(self):
+ new = object.__new__(type(self))
+ new.__setstate__(self.__dict__.copy())
+ return new
+
+ def __deepcopy__(self, memo):
+ new = object.__new__(type(self))
+ memo[id(self)] = new
+ new.__setstate__(copy.deepcopy(self.__dict__, memo))
+ return new
+
+ qualname = "_RawRead_" + "_".join(f"{cls.__module__}.{cls.__qualname__}".split("."))
+ sub = type(
+ cls.__name__,
+ (cls,),
+ {
+ "weight": property(weight),
+ "__reduce_ex__": __reduce_ex__,
+ "__copy__": __copy__,
+ "__deepcopy__": __deepcopy__,
+ "__module__": __name__,
+ "__qualname__": qualname,
+ _MARK: True,
+ },
+ )
+ globals()[qualname] = sub # resolvable by qualname (torch.compile, introspection)
+ _RAW_READ_CLASSES[cls] = sub
+ return sub
+
+
+def _install_raw_read(mixer, leaf):
+ """Give the adapted leaf the effective-weight ``weight``; False when it cannot take one."""
+ module = mixer._modules.get(leaf)
+ try:
+ from peft.tuners.tuners_utils import BaseTunerLayer
+ except Exception:
+ return False
+ if not isinstance(module, BaseTunerLayer):
+ return False
+ if getattr(type(module), _MARK, False):
+ return True
+ if torch.compiler.is_compiling():
+ # Dynamo cannot trace the class swap; skipping it would silently drop the adapter, so fail loudly.
+ raise RuntimeError(
+ f"Unsloth: {type(mixer).__name__}.{leaf} has an active LoRA adapter that was attached "
+ "after Unsloth prepared this Mamba model, so a compiled forward would read its base "
+ "weight and drop the adapter. Run one eager forward, or call "
+ "unsloth.models._mamba_lora.patch_mamba_fused_lora(model), before torch.compile."
+ )
+ if not isinstance(inspect.getattr_static(type(module), "weight", None), property):
+ return False
+ module.__class__ = _raw_read_class(type(module))
+ return True
+
+
+def _prepare_leaves(mixer):
+ """Install the effective-weight class on every adapter leaf of a mixer, active or not."""
+ tuner = _TUNER_LAYER or _tuner_layer_class()
+ modules = mixer.__dict__.get("_modules") or {}
+ for leaf in _RAW_WEIGHT_LEAVES:
+ if tuner and isinstance(modules.get(leaf), tuner):
+ _install_raw_read(mixer, leaf)
def _swap(value, merged, used):
if isinstance(value, torch.Tensor):
- entry = merged.get(id(value))
- if entry is not None and entry[0] is value:
- used.append(entry[2])
- return entry[1]
+ for source, weight, leaf in merged:
+ if source is value:
+ used.append(leaf)
+ return weight
return value
@@ -268,63 +482,106 @@ def _patch_class(cls):
@functools.wraps(forward)
def forward_keep_adapters(self, hidden_states, *args, **kwargs):
- if self.training:
- if "cache_params" in kwargs:
- cache_params = kwargs["cache_params"]
+ leaves = mamba_raw_weight_adapters(self)
+ if not leaves:
+ return forward(self, hidden_states, *args, **kwargs)
+ if "cache_params" in kwargs:
+ cache_params = kwargs["cache_params"]
+ else:
+ cache_params = args[cache_index] if 0 <= cache_index < len(args) else None
+ compiling = torch.compiler.is_compiling()
+ readable = [leaf for leaf in leaves if _install_raw_read(self, leaf)]
+ raw, depth = _routing.raw, len(_routing.raw)
+ if not (self.training and cache_params is None):
+ # Eval / cached decode: the torch path reads dt_proj.weight raw; serve W + delta lazily.
+ batch = hidden_states.shape[0] if isinstance(hidden_states, torch.Tensor) else None
+ for leaf in readable:
+ raw.append((self._modules[leaf], None, f"{cls.__name__}.{leaf}", batch))
+ try:
+ return forward(self, hidden_states, *args, **kwargs)
+ finally:
+ while len(raw) > depth:
+ raw.pop()
+ names = _gate_module_fns(cls)
+ modules = {leaf: self._modules[leaf] for leaf in leaves}
+ weights = {leaf: _merged_weight(modules[leaf]) for leaf in leaves} if names else {}
+ fused_cover = [leaf for leaf in leaves if weights.get(leaf) is not None]
+ # Leave the fused kernel only when the non-fused path applies more adapters.
+ use_slow = sum(leaf in called for leaf in leaves) > len(fused_cover)
+ merged = []
+ frame = _Frame(merged)
+ for leaf in fused_cover:
+ module = modules[leaf]
+ if leaf in readable:
+ # `.weight` now returns this tensor, so the torch path's raw read applies it too.
+ raw.append((module, weights[leaf], f"{cls.__name__}.{leaf}", None))
+ frame.applied.add(leaf)
+ if not use_slow:
+ base = module.get_base_layer().weight
+ merged.append((base, weights[leaf], leaf))
+ merged.append((weights[leaf], weights[leaf], leaf))
+ hooks = []
+ if not compiling:
+ hooks = [
+ module.register_forward_pre_hook(
+ functools.partial(_mark_applied, frame.applied, leaf)
+ )
+ for leaf, module in modules.items()
+ if leaf not in frame.applied
+ ]
+ _routing.stack.append(frame)
+ try:
+ if use_slow:
+ out = non_fused(self, hidden_states, *args, **kwargs)
else:
- cache_params = args[cache_index] if 0 <= cache_index < len(args) else None
- leaves = mamba_raw_weight_adapters(self) if cache_params is None else None
- if leaves:
- names = _gate_module_fns(cls)
- modules = {leaf: self._modules[leaf] for leaf in leaves}
- weights = {leaf: _merged_weight(modules[leaf]) for leaf in leaves} if names else {}
- fused_cover = [leaf for leaf in leaves if weights.get(leaf) is not None]
- # Leave the fused kernel only when the non-fused path applies more adapters.
- use_slow = sum(leaf in called for leaf in leaves) > len(fused_cover)
- merged = {}
- if not use_slow:
- for leaf in fused_cover:
- base = modules[leaf].get_base_layer().weight
- merged[id(base)] = (base, weights[leaf], leaf)
- frame = _Frame(merged)
- hooks = [
- module.register_forward_pre_hook(
- functools.partial(_mark_applied, frame.applied, leaf)
- )
- for leaf, module in modules.items()
- ]
- _routing.stack.append(frame)
- try:
- if use_slow:
- out = non_fused(self, hidden_states, *args, **kwargs)
- else:
- # Fused kernel kept; it receives W + delta in place of each adapted W.
- out = forward(self, hidden_states, *args, **kwargs)
- finally:
- _routing.stack.pop()
- for hook in hooks:
- hook.remove()
- _warn_not_applied(cls, [leaf for leaf in leaves if leaf not in frame.applied])
- return out
- return forward(self, hidden_states, *args, **kwargs)
+ # Fused kernel kept; it receives W + delta in place of each adapted W.
+ out = forward(self, hidden_states, *args, **kwargs)
+ finally:
+ _routing.stack.pop()
+ for hook in hooks:
+ hook.remove()
+ while len(raw) > depth:
+ raw.pop()
+ if not compiling:
+ _warn_not_applied(cls, [leaf for leaf in leaves if leaf not in frame.applied])
+ return out
setattr(forward_keep_adapters, _MARK, True)
cls.forward = forward_keep_adapters
+ _prepare_on_assign(cls)
return True
+def _prepare_on_assign(cls):
+ # PEFT attaches adapters via setattr on the mixer: install the class there, before any compile.
+ original = cls.__setattr__
+ if getattr(original, _MARK, False):
+ return
+
+ @functools.wraps(original)
+ def __setattr__(self, name, value):
+ original(self, name, value)
+ if name in _RAW_WEIGHT_LEAVES and not torch.compiler.is_compiling():
+ _prepare_leaves(self)
+
+ setattr(__setattr__, _MARK, True)
+ cls.__setattr__ = __setattr__
+
+
def patch_mamba_fused_lora(model):
"""Keep LoRA on Mamba mixers whose fused kernels read x_proj / dt_proj / out_proj raw."""
if not isinstance(model, torch.nn.Module):
return model
- seen = set()
+ patched = {}
for module in model.modules():
cls = type(module)
- if cls in seen or not _is_mamba_mixer(module):
+ if not _is_mamba_mixer(module):
continue
- seen.add(cls)
- try:
- _patch_class(cls)
- except Exception:
- pass
+ if cls not in patched:
+ try:
+ patched[cls] = _patch_class(cls)
+ except Exception:
+ patched[cls] = False
+ if patched[cls]:
+ _prepare_leaves(module)
return model