Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Mamba LoRA: apply a trained dt_proj / x_proj / out_proj adapter in eval and cached decode (#15730)
Daniel Han802c5898c291
diff --git a/tests/test_mamba_dt_proj_eval_parity.py b/tests/test_mamba_dt_proj_eval_parity.pynew file mode 100644index 00000000000..4d11ecbf714--- /dev/null+++ b/tests/test_mamba_dt_proj_eval_parity.py@@ -0,0 +1,676 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A trained LoRA on Mamba's dt_proj must also apply in eval, cached decode and generate.++transformers' Mamba torch path (transformers 5 always, 4.x ``cuda_kernels_forward`` outside fused+training) computes ``self.dt_proj.weight @ time_step``, which reads the base weight and skips the+adapter. Unmerged eval / generate must equal the ``merge_and_unload`` model, ``disable_adapter()``+must give the base model, and the fused training path must stay compile friendly.+"""++import copy+import importlib+import importlib.util+import pathlib+import pickle+import subprocess+import sys+import threading+import warnings++import pytest+import torch+import torch.nn.functional as F++peft = pytest.importorskip("peft")++_HERE = pathlib.Path(__file__).resolve().parent+_spec = importlib.util.spec_from_file_location(+ "_mamba_fused_lora_helpers", _HERE / "test_mamba_fused_lora_adapters.py"+)+helpers = importlib.util.module_from_spec(_spec)+sys.modules[_spec.name] = helpers # torch.compile resolves the fake kernel's globals by module+_spec.loader.exec_module(helpers)++ARCHS = list(helpers.ARCHS)+ALL = ["in_proj", "x_proj", "dt_proj"]+++# transformers 4.x: the test models always dispatch to cuda_kernels_forward, which outside fused+# training reads dt_proj.weight raw. Its mamba_ssm / causal_conv1d ops get their reference math here+# so prefill and cached decode run on CPU.+def _causal_conv1d_ref(+ x,+ weight,+ bias = None,+ activation = None,+ **kwargs,+):+ seq_len = x.shape[-1]+ out = F.conv1d(x, weight.unsqueeze(1), bias, padding = weight.shape[-1] - 1, groups = x.shape[1])+ out = out[..., :seq_len]+ return F.silu(out) if activation in ("silu", "swish") else out+++def _selective_scan_ref(+ u,+ delta,+ A,+ B,+ C,+ D = None,+ z = None,+ delta_bias = None,+ delta_softplus = False,+ return_last_state = False,+):+ delta = delta.float()+ if delta_bias is not None:+ delta = delta + delta_bias[None, :, None].float()+ if delta_softplus:+ delta = F.softplus(delta)+ state = u.new_zeros(u.shape[0], u.shape[1], A.shape[1], dtype = torch.float32)+ ys = []+ for t in range(u.shape[-1]):+ state = (+ torch.exp(delta[:, :, t, None] * A) * state+ + delta[:, :, t, None] * B[:, None, :, t] * u[:, :, t, None]+ )+ ys.append((state * C[:, None, :, t]).sum(-1))+ y = torch.stack(ys, dim = -1)+ if D is not None:+ y = y + u * D[None, :, None]+ if z is not None:+ y = y * F.silu(z)+ y = y.to(u.dtype)+ return (y, state) if return_last_state else y+++def _causal_conv1d_update_ref(+ x,+ conv_state,+ weight,+ bias = None,+ activation = None,+ **kwargs,+):+ conv_state.copy_(torch.roll(conv_state, shifts = -1, dims = -1))+ conv_state[:, :, -1] = x+ out = (conv_state * weight).sum(-1)+ if bias is not None:+ out = out + bias+ return F.silu(out) if activation in ("silu", "swish") else out+++def _selective_state_update_ref(+ state,+ x,+ dt,+ A,+ B,+ C,+ D = None,+ z = None,+ dt_bias = None,+ dt_softplus = False,+ **kwargs,+):+ dt = dt.float()+ if dt_bias is not None:+ dt = dt + dt_bias.float()+ if dt_softplus:+ dt = F.softplus(dt)+ new = torch.exp(dt[..., None] * A) * state + dt[..., None] * B[:, None, :] * x[..., None]+ state.copy_(new)+ y = (new * C[:, None, :]).sum(-1)+ if D is not None:+ y = y + x * D+ if z is not None:+ y = y * F.silu(z)+ return y.to(x.dtype)+++def _install_legacy_reference_ops(monkeypatch, arch):+ modeling = importlib.import_module(f"transformers.models.{arch}.modeling_{arch}")+ ops = {+ "selective_scan_fn": _selective_scan_ref,+ "selective_state_update": _selective_state_update_ref,+ "_lazy_load_causal_conv1d": lambda: (_causal_conv1d_update_ref, _causal_conv1d_ref),+ # transformers < 4.56 binds the causal_conv1d ops as module globals instead of lazily.+ "causal_conv1d_fn": _causal_conv1d_ref,+ "causal_conv1d_update": _causal_conv1d_update_ref,+ }+ for name, fn in ops.items():+ monkeypatch.setattr(modeling, name, fn, raising = False)+++def _trained(monkeypatch, arch, targets, **kwargs):+ """A model whose adapters took one step through the fused training path, then put in eval."""+ model, calls = helpers._build(+ monkeypatch, arch, targets, kernel = helpers._reference_fused, **kwargs+ )+ ids = torch.randint(0, 64, (2, 8), generator = torch.Generator().manual_seed(5))+ helpers._loss(model, ids).backward()+ assert calls, "training did not take the fused path"+ grads = [p.grad for p in helpers._lora_param(model, "dt_proj", "B")]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)+ with torch.no_grad():+ for name, p in model.named_parameters():+ if p.grad is not None:+ p -= 10.0 * p.grad+ model.zero_grad(set_to_none = True)+ model.eval()+ if helpers._legacy(arch):+ _install_legacy_reference_ops(monkeypatch, arch)+ return model+++def _merged(model):+ return copy.deepcopy(model).merge_and_unload().eval()+++def _logits(model, ids):+ with torch.no_grad():+ return model(input_ids = ids, use_cache = False).logits+++def _ids(seed = 7):+ return torch.randint(0, 64, (2, 12), generator = torch.Generator().manual_seed(seed))++[email protected]("arch", ARCHS)[email protected]("targets", [["dt_proj"], ALL], ids = ["dt_proj", "all"])+def test_eval_logits_match_merged(monkeypatch, arch, targets):+ model = _trained(monkeypatch, arch, targets)+ ids = _ids()+ unmerged, merged = _logits(model, ids), _logits(_merged(model), ids)+ with model.disable_adapter():+ base = _logits(model, ids)+ assert (merged - base).abs().max() > 1e-6, "the trained adapter has no effect"+ if targets == ["dt_proj"]:+ # dt_proj is only read as a weight, so W + delta is the merged weight bit for bit.+ assert torch.equal(unmerged, merged), (unmerged - merged).abs().max()+ else:+ torch.testing.assert_close(unmerged, merged, rtol = 1e-5, atol = 1e-5)++[email protected]("arch", ARCHS)+def test_greedy_generate_matches_merged(monkeypatch, arch):+ model = _trained(monkeypatch, arch, ALL)+ merged = _merged(model)+ ids = _ids(11)[:1, :6]+ kwargs = dict(+ max_new_tokens = 8,+ do_sample = False,+ use_cache = True,+ output_scores = True,+ return_dict_in_generate = True,+ pad_token_id = 0,+ )+ with torch.no_grad():+ out_u = model.generate(input_ids = ids, **kwargs)+ out_m = merged.generate(input_ids = ids, **kwargs)+ assert torch.equal(out_u.sequences, out_m.sequences)+ for su, sm in zip(out_u.scores, out_m.scores):+ torch.testing.assert_close(su, sm, rtol = 1e-5, atol = 1e-5)++[email protected]("arch", ARCHS)+def test_disable_adapter_gives_base_model(monkeypatch, arch):+ model = _trained(monkeypatch, arch, ALL)+ ids = _ids()+ base = _logits(copy.deepcopy(model).unload().eval(), ids)+ with model.disable_adapter():+ assert torch.equal(_logits(model, ids), base)+ assert not torch.equal(_logits(model, ids), base)++[email protected]("arch", ARCHS)+def test_eval_weight_follows_updates_and_adapter_switches(monkeypatch, arch):+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids()+ first = _logits(model, ids)+ # An in-place optimizer-style update must show in the next eval forward.+ with torch.no_grad():+ for p in helpers._lora_param(model, "dt_proj", "B"):+ p.mul_(-1.5)+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))+ assert not torch.equal(_logits(model, ids), first)++ model.add_adapter("second", peft.LoraConfig(r = 2, lora_alpha = 4, target_modules = ["dt_proj"]))+ with torch.no_grad():+ for name, p in model.named_parameters():+ if ".lora_B.second." in name:+ p.normal_(std = 0.5)+ model.eval()+ for active in ("second", "default"):+ model.set_adapter(active)+ model.eval()+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids)), active+ model.base_model.set_adapter(["default", "second"])+ model.eval()+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))++ # merge_adapter() folds the delta into the base weight: no double counting, and unmerge restores.+ unmerged = _logits(model, ids)+ model.merge_adapter()+ torch.testing.assert_close(_logits(model, ids), unmerged, rtol = 1e-5, atol = 1e-6)+ model.unmerge_adapter()+ torch.testing.assert_close(_logits(model, ids), unmerged, rtol = 1e-5, atol = 1e-6)++[email protected]("arch", ARCHS)+def test_eval_weight_follows_data_reassignment(monkeypatch, arch):+ # `p.data = t` (custom LoRA inits, dtype casts at save time) keeps id(p) and p._version.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids()+ first = _logits(model, ids)+ for p in helpers._lora_param(model, "dt_proj", "B"):+ p.data = p.data * -1.5+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))+ assert not torch.equal(_logits(model, ids), first)++[email protected]("arch", ARCHS)+def test_eval_weight_follows_dtype_conversion(monkeypatch, arch):+ # model.double() / .bfloat16() keep the parameter objects and their version counters.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids()+ _logits(model, ids)+ model.double()+ assert all(p.dtype == torch.float64 for p in helpers._lora_param(model, "dt_proj", "B"))+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))++[email protected]("arch", ARCHS)+def test_eval_weight_follows_unversioned_optimizer_step(monkeypatch, arch):+ # FlashAdamW's Triton update writes the parameters in place without bumping their version+ # counters (as an in-place op through `p.data` does), so eval -> train step -> eval must not+ # serve the W + delta of the first eval.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids()+ first = _logits(model, ids)+ model.train()+ helpers._loss(model, ids).backward()+ for p in helpers._lora_param(model, "dt_proj", "B"):+ version = p._version+ p.data.add_(p.grad.data, alpha = -10.0)+ p.data.mul_(-1.5)+ assert p._version == version+ model.zero_grad(set_to_none = True)+ model.eval()+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))+ assert not torch.equal(_logits(model, ids), first)++[email protected]("arch", ARCHS)+def test_eval_holds_no_effective_weight_copy(monkeypatch, arch):+ # W + delta is rebuilt per read, never stored on the leaf, so model.cpu() / .to() leave no+ # stale copy behind and every later read sees the converted weights.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids()+ _logits(model, ids)+ leaves = [m for m in model.modules() if type(m).__name__ == "Linear" and hasattr(m, "lora_A")]+ assert leaves+ for leaf in leaves:+ assert not any(isinstance(v, torch.Tensor) for v in leaf.__dict__.values())+ assert not any(+ isinstance(v, tuple) and any(isinstance(t, torch.Tensor) for t in v)+ for v in leaf.__dict__.values()+ )+ model.to(torch.float64)+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))++[email protected]("arch", ARCHS)+def test_autocast_eval_does_not_leak_into_full_precision(monkeypatch, arch):+ # W + delta is built outside autocast: no bf16-rounded delta from an earlier autocast forward.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids()+ with torch.autocast("cpu", dtype = torch.bfloat16):+ _logits(model, ids)+ assert torch.equal(_logits(model, ids), _logits(_merged(model), ids))++[email protected]("arch", ARCHS)+def test_mixed_adapter_batch_routes_each_row(monkeypatch, arch):+ # PEFT's adapter_names= picks an adapter per row ("__base__" = none) through LoRA pre-hooks that a+ # raw weight read bypasses: each row must still get exactly its own adapter.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids()+ adapted = _logits(model, ids)+ with model.disable_adapter():+ base = _logits(model, ids)+ assert (adapted - base).abs().max() > 1e-6, "the trained adapter has no effect"+ with torch.no_grad():+ mixed = model(input_ids = ids, use_cache = False, adapter_names = ["__base__", "default"]).logits+ torch.testing.assert_close(mixed[0], base[0], rtol = 1e-6, atol = 1e-6)+ torch.testing.assert_close(mixed[1], adapted[1], rtol = 1e-6, atol = 1e-6)+ # The mixed batch must not poison the plain path afterwards.+ assert torch.equal(_logits(model, ids), adapted)+++def test_concurrent_eval_threads_each_see_the_adapter(monkeypatch):+ # nn.DataParallel replicas and serving threads run forwards concurrently: the effective weight+ # is armed per thread, so one thread finishing cannot disarm another mid-forward.+ model = _trained(monkeypatch, "mamba", ["dt_proj"])+ ids = _ids()+ expected = _logits(_merged(model), ids)+ results, errors = [], []++ def run():+ try:+ for _ in range(5):+ results.append(torch.equal(_logits(model, ids), expected))+ except BaseException as e:+ errors.append(e)++ threads = [threading.Thread(target = run) for _ in range(4)]+ for thread in threads:+ thread.start()+ for thread in threads:+ thread.join(120)+ if errors:+ raise errors[0]+ assert len(results) == 20 and all(results)+++def test_pickled_model_loads_without_unsloth(monkeypatch, tmp_path):+ # torch.save(model) / pickle / deepcopy must stay loadable in a process that never ran the patch.+ model = _trained(monkeypatch, "mamba", ["dt_proj"])+ ids = _ids()+ expected = _logits(model, ids)+ leaf = model.base_model.model.backbone.layers[0].mixer.dt_proj+ data = pickle.dumps(model)+ assert b"_RawRead_" not in data+ # copy / deepcopy keep the effective-weight class (see the compiled deepcopy test below).+ assert type(copy.deepcopy(leaf)) is type(leaf)+ assert torch.equal(_logits(pickle.loads(data), ids), expected)+ path = tmp_path / "model.pkl"+ path.write_bytes(data)+ code = (+ "import pickle, sys; m = pickle.load(open(sys.argv[1], 'rb')); "+ "print(type(m.base_model.model.backbone.layers[0].mixer.dt_proj).__qualname__)"+ )+ out = subprocess.run([sys.executable, "-c", code, str(path)], capture_output = True, text = True)+ assert out.returncode == 0, out.stderr[-2000:]+ assert out.stdout.strip() == type(leaf).__mro__[1].__qualname__+++def test_unfoldable_adapter_warns_on_raw_read(monkeypatch):+ if helpers._legacy("mamba"):+ pytest.skip("transformers 4.x eval on CPU calls dt_proj as a module")+ model, _ = helpers._build(monkeypatch, "mamba", ["dt_proj"], fused = False)+ model = peft.get_peft_model(+ model.unload(),+ peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["dt_proj"], use_dora = True),+ )+ helpers._patch_fn()(model)+ model.eval()+ with pytest.warns(UserWarning, match = r"MambaMixer\.dt_proj is not applied"):+ _logits(model, _ids())++[email protected]("arch", ARCHS)+def test_fused_training_mixer_compiles_fullgraph(monkeypatch, arch):+ model, calls = helpers._build(monkeypatch, arch, ALL, kernel = helpers._reference_fused)+ mixer = helpers._mixers(model)[0]+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))+ params = [p for n, p in mixer.named_parameters() if ".lora_" in n]++ def run(fn):+ for p in params:+ p.grad = None+ out = fn(hidden)+ out.float().pow(2).sum().backward()+ return out.detach(), [p.grad.clone() for p in params]++ out_e, grads_e = run(mixer)+ torch._dynamo.reset()+ compiled = torch.compile(mixer, backend = "aot_eager", fullgraph = True)+ with warnings.catch_warnings():+ warnings.simplefilter("ignore")+ out_c, grads_c = run(compiled)+ run(compiled)+ torch._dynamo.reset()+ torch.testing.assert_close(out_c, out_e, rtol = 1e-5, atol = 1e-6)+ for gc, ge in zip(grads_c, grads_e):+ assert ge.abs().sum() > 0+ torch.testing.assert_close(gc, ge, rtol = 1e-4, atol = 1e-6)+++def _hotswap_fn():+ try:+ from peft.utils.hotswap import hotswap_adapter+ except Exception:+ return None+ return hotswap_adapter++[email protected]("arch", ARCHS)+def test_hotswapped_adapter_applies_in_eval(monkeypatch, arch, tmp_path):+ # PEFT's hotswap_adapter replaces same-shaped LoRA weights through `old.data.copy_(new)`, which+ # keeps each parameter's identity, storage and version counter; eval must serve the new adapter.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids()+ other = copy.deepcopy(model)+ with torch.no_grad():+ for name, p in other.named_parameters():+ if ".lora_" in name:+ p.normal_(std = 2.0, generator = torch.Generator().manual_seed(3))+ expected = _logits(_merged(other), ids)+ old = _logits(model, ids) # an eval forward before the swap+ assert (expected - old).abs().max() > 1e-3, "the replacement adapter has no effect"+ hotswap = _hotswap_fn()+ if hotswap is not None:+ other.save_pretrained(tmp_path)+ hotswap(model, str(tmp_path), adapter_name = "default", torch_device = "cpu")+ else:+ new = dict(other.named_parameters())+ for name, p in model.named_parameters():+ if ".lora_" in name:+ p.data.copy_(new[name].data)+ # No model.eval() / train() here: a hot swap in a serving process calls neither.+ assert torch.equal(_logits(model, ids), expected)++[email protected]("arch", ARCHS)+def test_eval_mixer_compiles_fullgraph_under_no_grad(monkeypatch, arch):+ # A no-grad eval forward reads dt_proj.weight raw; under torch.compile it must trace in one+ # graph (no cache lookups or mutations inside it) and match eager.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ with torch.no_grad():+ for name, p in model.named_parameters():+ if ".lora_" in name:+ p.normal_(std = 2.0, generator = torch.Generator().manual_seed(3))+ mixer = helpers._mixers(model)[0]+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))+ with torch.no_grad():+ eager = mixer(hidden)+ with model.disable_adapter():+ base = mixer(hidden)+ assert (eager - base).abs().max() > 1e-5, "the adapter has no effect"+ torch._dynamo.reset()+ try:+ compiled = torch.compile(mixer, backend = "aot_eager", fullgraph = True)+ with warnings.catch_warnings(), torch.no_grad():+ warnings.simplefilter("ignore")+ out = compiled(hidden)+ again = compiled(hidden)+ finally:+ torch._dynamo.reset()+ torch.testing.assert_close(out, eager, rtol = 1e-5, atol = 1e-6)+ assert torch.equal(out, again)+++def _adapted_mixer(monkeypatch, arch):+ model = _trained(monkeypatch, arch, ["dt_proj"])+ with torch.no_grad():+ for name, p in model.named_parameters():+ if ".lora_" in name:+ p.normal_(std = 2.0, generator = torch.Generator().manual_seed(3))+ return model, helpers._mixers(model)[0]+++def _compiled_no_grad(+ mixer,+ hidden,+ fullgraph = True,+):+ torch._dynamo.reset()+ try:+ compiled = torch.compile(mixer, backend = "aot_eager", fullgraph = fullgraph)+ with warnings.catch_warnings(), torch.no_grad():+ warnings.simplefilter("ignore")+ out = compiled(hidden)+ again = compiled(hidden)+ finally:+ torch._dynamo.reset()+ assert torch.equal(out, again)+ return out+++def _eager_and_base(model, mixer, hidden):+ with torch.no_grad():+ eager = mixer(hidden)+ with model.disable_adapter():+ base = mixer(hidden)+ assert (eager - base).abs().max() > 1e-5, "the adapter has no effect"+ return eager++[email protected]("arch", ARCHS)+def test_deepcopied_eval_mixer_compiles_with_its_adapter(monkeypatch, arch):+ # A deep copy compiled before any eager forward must still serve W + delta, not the base weight.+ model, mixer = _adapted_mixer(monkeypatch, arch)+ copied = copy.deepcopy(mixer)+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))+ out = _compiled_no_grad(copied, hidden)+ torch.testing.assert_close(out, _eager_and_base(model, mixer, hidden), rtol = 1e-5, atol = 1e-6)+ assert type(copied.dt_proj) is type(mixer.dt_proj)++[email protected]("arch", ARCHS)+def test_adapter_attached_after_patch_compiles_with_it(monkeypatch, arch):+ # A dt_proj adapter added after the patch, compiled before any eager forward, must apply.+ model, _ = helpers._build(monkeypatch, arch, ["x_proj"], kernel = helpers._reference_fused)+ if helpers._legacy(arch):+ _install_legacy_reference_ops(monkeypatch, arch)+ model.add_adapter("late", peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["dt_proj"]))+ model.set_adapter("late")+ model.eval()+ with torch.no_grad():+ for name, p in model.named_parameters():+ if ".lora_" in name and ".late." in name:+ p.normal_(std = 2.0, generator = torch.Generator().manual_seed(3))+ mixer = helpers._mixers(model)[0]+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))+ out = _compiled_no_grad(mixer, hidden)+ torch.testing.assert_close(out, _eager_and_base(model, mixer, hidden), rtol = 1e-5, atol = 1e-6)++[email protected]("arch", ARCHS)+def test_unprepared_adapter_never_compiles_silently(monkeypatch, arch):+ # A leaf that lost its effective-weight class: fullgraph fails loudly, otherwise the frame runs+ # eagerly (which installs it) and every call applies the adapter.+ model, mixer = _adapted_mixer(monkeypatch, arch)+ hidden = torch.randn(2, 8, 16, generator = torch.Generator().manual_seed(2))+ eager = _eager_and_base(model, mixer, hidden)+ plain = type(mixer.dt_proj).__mro__[1]+ mixer.dt_proj.__class__ = plain+ with pytest.raises(Exception, match = r"dt_proj has an active LoRA adapter"):+ _compiled_no_grad(mixer, hidden)+ mixer.dt_proj.__class__ = plain+ out = _compiled_no_grad(mixer, hidden, fullgraph = False)+ torch.testing.assert_close(out, eager, rtol = 1e-5, atol = 1e-6)+++def _row_logits(model, sequences, prompt_len, names):+ """Per-step logits of each generated row, recomputed with that row's adapter alone."""+ out = []+ for row, name in zip(sequences, names):+ if name == "__base__":+ with model.disable_adapter():+ logits = _logits(model, row[None])+ else:+ logits = _logits(model, row[None])+ out.append(logits[0, prompt_len - 1 : -1])+ return torch.stack(out)++[email protected]("arch", ARCHS)+ "names", [["__base__", "default"], ["default", "default"]], ids = ["mixed", "same"]+)[email protected]("greedy", [False, True], ids = ["sample", "greedy"])+def test_mixed_adapter_generate_num_return_sequences(monkeypatch, arch, names, greedy):+ # generate expands each prompt num_return_sequences times (repeat_interleave), while PEFT's+ # adapter_names hook keeps one name per prompt: every expanded row keeps its prompt's adapter.+ # transformers refuses num_return_sequences without sampling, so greedy is top_k = 1 sampling.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids(13)[:, :4]+ torch.manual_seed(0)+ with torch.no_grad():+ out = model.generate(+ input_ids = ids,+ adapter_names = names,+ do_sample = True,+ top_k = 1 if greedy else 0,+ num_return_sequences = 2,+ max_new_tokens = 3,+ use_cache = True,+ output_logits = True,+ return_dict_in_generate = True,+ pad_token_id = 0,+ )+ assert out.sequences.shape[0] == 4+ row_names = [name for name in names for _ in range(2)]+ assert torch.equal(out.sequences[:, :4], ids.repeat_interleave(2, dim = 0))+ got = torch.stack(out.logits, dim = 1)+ expected = _row_logits(model, out.sequences, 4, row_names)+ torch.testing.assert_close(got, expected, rtol = 1e-5, atol = 1e-5)+ if greedy:+ for i, name in enumerate(names):+ kwargs = dict(max_new_tokens = 3, do_sample = False, use_cache = True, pad_token_id = 0)+ with torch.no_grad():+ if name == "__base__":+ with model.disable_adapter():+ alone = model.generate(input_ids = ids[i : i + 1], **kwargs)+ else:+ alone = model.generate(input_ids = ids[i : i + 1], **kwargs)+ assert torch.equal(out.sequences[2 * i], alone[0])+ assert torch.equal(out.sequences[2 * i + 1], alone[0])++[email protected]("arch", ARCHS)+def test_mixed_adapter_generate_beam_search(monkeypatch, arch):+ # PEFT repeats adapter_names num_beams times itself; they must be used as given.+ model = _trained(monkeypatch, arch, ["dt_proj"])+ ids = _ids(17)[:, :4]+ names = ["__base__", "default"]+ kwargs = dict(+ max_new_tokens = 3,+ num_beams = 2,+ num_return_sequences = 2,+ do_sample = False,+ use_cache = True,+ pad_token_id = 0,+ )+ with torch.no_grad():+ out = model.generate(input_ids = ids, adapter_names = names, **kwargs)+ assert out.shape[0] == 4+ for i, name in enumerate(names):+ if name == "__base__":+ with model.disable_adapter():+ alone = model.generate(input_ids = ids[i : i + 1], **kwargs)+ else:+ alone = model.generate(input_ids = ids[i : i + 1], **kwargs)+ assert torch.equal(out[2 * i : 2 * i + 2], alone)+++def test_mixed_adapter_names_not_dividing_batch_raise(monkeypatch):+ model = _trained(monkeypatch, "mamba", ["dt_proj"])+ ids = torch.randint(0, 64, (3, 6), generator = torch.Generator().manual_seed(3))+ with torch.no_grad(), pytest.raises(ValueError, match = "adapter_names"):+ model(input_ids = ids, use_cache = False, adapter_names = ["__base__", "default"])diff --git a/tests/test_mamba_fused_lora_adapters.py b/tests/test_mamba_fused_lora_adapters.pyindex 504a6459ef5..822eb2b655c 100644--- a/tests/test_mamba_fused_lora_adapters.py+++ b/tests/test_mamba_fused_lora_adapters.py@@ -14,6 +14,7 @@ import importlibimport importlib.utilimport inspectimport pathlib+import sysimport threadingimport warnings@@ -36,12 +37,17 @@ _FIX = _ROOT / "unsloth" / "models" / "_mamba_lora.py"def _patch_fn():- # Loaded by path (torch + peft only); a no-op on trees without the fix.+ # Loaded by path (torch + peft only), once, like a real import; a no-op on trees without the fix.if not _FIX.is_file():return lambda model: model- spec = importlib.util.spec_from_file_location("_unsloth_mamba_lora_under_test", _FIX)- module = importlib.util.module_from_spec(spec)- spec.loader.exec_module(module)+ name = "_unsloth_mamba_lora_under_test"+ module = sys.modules.get(name)+ if module is None:+ spec = importlib.util.spec_from_file_location(name, _FIX)+ module = importlib.util.module_from_spec(spec)+ # torch.compile resolves classes through sys.modules.+ sys.modules[name] = module+ spec.loader.exec_module(module)return module.patch_mamba_fused_lora@@ -142,6 +148,7 @@ def _build(monkeypatch.setattr(modeling, "mamba_inner_fn", mamba_inner_fn)monkeypatch.setattr(mixer_cls, "forward", mixer_cls.forward) # undo what the fix installs+ monkeypatch.setattr(mixer_cls, "__setattr__", mixer_cls.__setattr__)if hasattr(mixer_cls, "cuda_kernels_forward"):# transformers 4.x dispatches to cuda_kernels_forward only on CUDA with mamba_ssm; force it,# or its torch path for the non-fused reference.@@ -402,18 +409,20 @@ def test_unmergeable_dt_proj_routes_only_where_the_slow_path_calls_it(monkeypatcassert trained != warned, (trained, warned)-def test_dt_proj_lora_without_kernel_on_transformers_5_warns(monkeypatch):+def test_dt_proj_lora_without_kernel_on_transformers_5_trains(monkeypatch):# No fused kernel (the entry point returns None): transformers 5 falls through to its torch path,- # which reads dt_proj.weight raw, so even plain LoRA there cannot train.+ # which reads dt_proj.weight raw; the effective-weight property makes that read W + delta.if _legacy("mamba"):pytest.skip("transformers 4.x slow_forward calls dt_proj as a module")model, calls = _build(monkeypatch, "mamba", ["in_proj", "x_proj", "dt_proj"], fused = False)- with pytest.warns(UserWarning, match = r"MambaMixer\.dt_proj") as caught:+ with warnings.catch_warnings(record = True) as caught:+ warnings.simplefilter("always")_loss(model, torch.randint(0, 64, (2, 8))).backward()- assert not any("x_proj" in str(w.message) for w in caught)+ assert not any("not applied" in str(w.message) for w in caught)assert len(calls) == 2- grads = [p.grad for p in _lora_param(model, "x_proj", "B")]- assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)+ for leaf in ("x_proj", "dt_proj"):+ grads = [p.grad for p in _lora_param(model, leaf, "B")]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads), leaf@pytest.mark.parametrize("arch", list(ARCHS))diff --git a/unsloth/models/_mamba_lora.py b/unsloth/models/_mamba_lora.pyindex 6c15c5eb8aa..7d28805bef3 100644--- a/unsloth/models/_mamba_lora.py+++ b/unsloth/models/_mamba_lora.py@@ -12,11 +12,18 @@ When every such adapter is plain LoRA (one active adapter, no dropout, no DoRA /bias, dense ``nn.Linear`` base) the fused kernel is kept and receives ``W + delta`` instead of ``W``,with ``delta`` from PEFT's own ``get_delta_weight`` built inside autograd, so lora_A / lora_B getexact gradients at fused-kernel memory and speed. Otherwise the call routes to the mixer's own-non-fused path when that path calls more of the adapted modules (transformers 5's Mamba torch path-still reads dt_proj.weight raw, so it does not help a dt_proj adapter). An adapter neither path-applies warns once. Mixers without raw-weight adapters are unchanged.+non-fused path when that path calls more of the adapted modules. An adapter neither path applies+warns once. Mixers without raw-weight adapters are unchanged.++The torch path (transformers 5 always, 4.x ``cuda_kernels_forward`` outside fused training) also+reads ``self.dt_proj.weight @ time_step`` raw, in eval and cached decode too. Adapted leaves therefore+get a ``weight`` property that, while their mixer's forward runs, returns ``W + delta`` (built inside+autograd and outside autocast, rebuilt on every read so no weight write can leave it stale), so+unmerged eval and generate match ``merge_and_unload`` and ``disable_adapter()`` gives the base model."""+import copy+import copyregimport functoolsimport inspectimport sys@@ -43,17 +50,22 @@ class _Routing(threading.local):def __init__(self):self.force_slow = 0self.stack = []+ # [(adapted leaf, W + delta or None = build on read, "Mixer.leaf", batch size or None)]+ self.raw = []_routing = _Routing()_warned = set()+_TUNER_LAYER = None+_RAW_READ_CLASSES = {}class _Frame:__slots__ = ("merged", "applied")def __init__(self, merged):- # {id(base weight): (base weight, W + delta, leaf)} handed to the fused kernel.+ # [(weight the forward passes, W + delta handed to the kernel instead, leaf)]; matched by+ # identity, not id(), which torch.compile cannot trace for a tensor built in the forward.self.merged = merged# Leaves whose adapter ran: merged into a kernel call that returned, or called as a module.self.applied = set()@@ -76,40 +88,62 @@ def _has_active_adapter(module):return False+def _tuner_layer_class():+ global _TUNER_LAYER+ if _TUNER_LAYER is None:+ try:+ from peft.tuners.tuners_utils import BaseTunerLayer+ except Exception:+ return ()+ _TUNER_LAYER = BaseTunerLayer+ return _TUNER_LAYER++def mamba_raw_weight_adapters(mixer):"""Names of the mixer's raw-weight leaves that carry an active, unmerged adapter."""- return [name for name in _RAW_WEIGHT_LEAVES if _has_active_adapter(mixer._modules.get(name))]+ modules, tuner = mixer._modules, _TUNER_LAYER or _tuner_layer_class()+ leaves = [name for name in _RAW_WEIGHT_LEAVES if isinstance(modules.get(name), tuner)]+ return [name for name in leaves if _has_active_adapter(modules[name])] if leaves else leaves-def _merged_weight(module):- """``W + delta`` when that is exactly what the LoRA module computes, else None."""+def _merged_weight(+ module,+ allow_multiple = False,+ names = None,+):+ """``W + delta`` when that is exactly what the LoRA module computes, else None.++ ``names`` replaces the active adapters (one row of a PEFT mixed-adapter batch).+ """try:from peft.tuners.lora import Linear as LoraLinearexcept Exception:return Noneif not isinstance(module, LoraLinear):return None- active = [name for name in module.active_adapters if name in module.lora_A]- if len(active) != 1:- return None- # Other adapter layers (embedding LoRA, a mixed tuner) for the active adapter are not covered.- for layer_name in module.adapter_layer_names:- if layer_name not in ("lora_A", "lora_B"):- layer = getattr(module, layer_name, None)- if layer is not None and active[0] in layer:- return None- name = active[0]- if name in getattr(module, "lora_variant", {}) or getattr(module, "use_dora", {}).get(name):+ if names is None:+ names = module.active_adapters+ active = [name for name in names if name in module.lora_A]+ if not active or (len(active) != 1 and not allow_multiple):return None+ for name in active:+ # Other adapter layers (embedding LoRA, a mixed tuner) for the active adapter are not covered.+ for layer_name in module.adapter_layer_names:+ if layer_name not in ("lora_A", "lora_B"):+ layer = getattr(module, layer_name, None)+ if layer is not None and name in layer:+ return None+ if name in getattr(module, "lora_variant", {}) or getattr(module, "use_dora", {}).get(name):+ return None+ if getattr(module.lora_B[name], "bias", None) is not None:+ return None+ dropout = module.lora_dropout[name]+ if not isinstance(dropout, (torch.nn.Identity, torch.nn.Dropout)):+ return None+ if isinstance(dropout, torch.nn.Dropout) and dropout.training and dropout.p > 0:+ return Noneif getattr(module, "fan_in_fan_out", False):return None- if getattr(module.lora_B[name], "bias", None) is not None:- return None- dropout = module.lora_dropout[name]- if not isinstance(dropout, (torch.nn.Identity, torch.nn.Dropout)):- return None- if isinstance(dropout, torch.nn.Dropout) and dropout.training and dropout.p > 0:- return Nonebase = module.get_base_layer()# Exact nn.Linear only: bitsandbytes / other quantized subclasses do not hold a dense weight.if type(base) is not torch.nn.Linear:@@ -117,15 +151,195 @@ def _merged_weight(module):weight = base.weightif not weight.is_floating_point() or weight.is_meta:return None- return weight + module.get_delta_weight(name).to(weight.dtype)+ # Same order as PEFT's merge (base += delta per adapter), so merged and unmerged match bitwise.+ for name in active:+ weight = weight + module.get_delta_weight(name).to(weight.dtype)+ return weight+++def _full_precision_merged(module, names = None):+ # Built outside autocast, as merge_and_unload does, so autocast eval matches the merged model.+ device_type = module.get_base_layer().weight.device.type+ try:+ # Autocast off already: same tensor without the context's cost, which shows on decode reads.+ context = None+ if torch.compiler.is_compiling() or torch.is_autocast_enabled(device_type):+ context = torch.autocast(device_type, enabled = False)+ except Exception:+ context = None+ if context is None:+ return _merged_weight(module, allow_multiple = True, names = names)+ with context:+ return _merged_weight(module, allow_multiple = True, names = names)+++def _mixed_batch_adapter_names(module):+ """Per-row adapter names of a PEFT mixed-adapter batch (``adapter_names=``), else None.++ PEFT delivers them through a forward pre-hook on each LoRA layer, which a raw ``.weight``+ read never runs.+ """+ for hook in module._forward_pre_hooks.values():+ if (+ isinstance(hook, functools.partial)+ and getattr(hook.func, "__name__", "") == "_adapter_names_pre_forward_hook"+ ):+ names = hook.keywords.get("adapter_names")+ if names is not None:+ return list(names)+ return None+++def _per_row_weight(+ module,+ names,+ label,+ batch = None,+):+ """[batch, out, in] effective weights, one per row of a mixed-adapter batch.++ ``W @ x`` with a [batch, in, seq] input broadcasts per row, so each row sees its own adapter and+ ``"__base__"`` / unknown names see the base weight, as PEFT's mixed-batch forward gives them.++ PEFT repeats the names only for ``num_beams`` (peft/tuners/lora/model.py+ ``_enable_peft_forward_hooks``); ``generate`` also expands each prompt ``num_return_sequences``+ times with ``repeat_interleave``, so a running batch that is a multiple of the names gets them+ repeated in that same order. Any other size cannot be routed and raises.+ """+ if batch is not None and batch != len(names):+ if not names or batch % len(names):+ raise ValueError(+ f"Unsloth: {label} got {len(names)} adapter_names for a batch of {batch} rows; "+ "pass one adapter name per input row."+ )+ repeat = batch // len(names)+ names = [name for name in names for _ in range(repeat)]+ base = module.get_base_layer().weight+ weights = {}+ for name in dict.fromkeys(names):+ if name == "__base__" or name not in module.lora_A:+ weights[name] = base+ continue+ weight = _full_precision_merged(module, names = [name])+ if weight is None:+ if not torch.compiler.is_compiling():+ _warn_raw_read(label)+ weight = base+ weights[name] = weight+ return torch.stack([weights[name] for name in names])+++def _warn_raw_read(label):+ if ("raw_read", label) in _warned:+ return+ _warned.add(("raw_read", label))+ warnings.warn(+ f"Unsloth: the adapter on {label} is not applied here: this Mamba forward reads that "+ "weight raw and only plain LoRA (no DoRA, no LoRA bias, no active dropout, dense base) can "+ "be folded into it. Merge the adapter (merge_and_unload) for inference.",+ stacklevel = 3,+ )+++def _raw_read_class(cls):+ """Subclass of an adapter layer whose ``weight`` is ``W + delta`` while its mixer forward runs."""+ sub = _RAW_READ_CLASSES.get(cls)+ if sub is not None:+ return sub++ def weight(self):+ # Per thread: nn.DataParallel replicas share their leaves' __dict__ contents.+ for module, effective, label, batch in reversed(_routing.raw):+ if module is not self:+ continue+ if effective is None:+ names = _mixed_batch_adapter_names(self)+ if names is not None:+ return _per_row_weight(self, names, label, batch)+ # Never cached: hotswap_adapter / FlashAdamW `.data` writes skip version counters.+ effective = _full_precision_merged(self)+ if effective is not None:+ return effective+ if not torch.compiler.is_compiling():+ _warn_raw_read(label)+ break+ return super(sub, self).weight++ def __reduce_ex__(self, protocol):+ # Pickle as the plain adapter layer so it loads without Unsloth.+ return copyreg._reconstructor, (cls, object, None), self.__dict__.copy()++ # copy / deepcopy keep this class: a trace cannot install it and would read the base weight.+ def __copy__(self):+ new = object.__new__(type(self))+ new.__setstate__(self.__dict__.copy())+ return new++ def __deepcopy__(self, memo):+ new = object.__new__(type(self))+ memo[id(self)] = new+ new.__setstate__(copy.deepcopy(self.__dict__, memo))+ return new++ qualname = "_RawRead_" + "_".join(f"{cls.__module__}.{cls.__qualname__}".split("."))+ sub = type(+ cls.__name__,+ (cls,),+ {+ "weight": property(weight),+ "__reduce_ex__": __reduce_ex__,+ "__copy__": __copy__,+ "__deepcopy__": __deepcopy__,+ "__module__": __name__,+ "__qualname__": qualname,+ _MARK: True,+ },+ )+ globals()[qualname] = sub # resolvable by qualname (torch.compile, introspection)+ _RAW_READ_CLASSES[cls] = sub+ return sub+++def _install_raw_read(mixer, leaf):+ """Give the adapted leaf the effective-weight ``weight``; False when it cannot take one."""+ module = mixer._modules.get(leaf)+ try:+ from peft.tuners.tuners_utils import BaseTunerLayer+ except Exception:+ return False+ if not isinstance(module, BaseTunerLayer):+ return False+ if getattr(type(module), _MARK, False):+ return True+ if torch.compiler.is_compiling():+ # Dynamo cannot trace the class swap; skipping it would silently drop the adapter, so fail loudly.+ raise RuntimeError(+ f"Unsloth: {type(mixer).__name__}.{leaf} has an active LoRA adapter that was attached "+ "after Unsloth prepared this Mamba model, so a compiled forward would read its base "+ "weight and drop the adapter. Run one eager forward, or call "+ "unsloth.models._mamba_lora.patch_mamba_fused_lora(model), before torch.compile."+ )+ if not isinstance(inspect.getattr_static(type(module), "weight", None), property):+ return False+ module.__class__ = _raw_read_class(type(module))+ return True+++def _prepare_leaves(mixer):+ """Install the effective-weight class on every adapter leaf of a mixer, active or not."""+ tuner = _TUNER_LAYER or _tuner_layer_class()+ modules = mixer.__dict__.get("_modules") or {}+ for leaf in _RAW_WEIGHT_LEAVES:+ if tuner and isinstance(modules.get(leaf), tuner):+ _install_raw_read(mixer, leaf)def _swap(value, merged, used):if isinstance(value, torch.Tensor):- entry = merged.get(id(value))- if entry is not None and entry[0] is value:- used.append(entry[2])- return entry[1]+ for source, weight, leaf in merged:+ if source is value:+ used.append(leaf)+ return weightreturn value@@ -268,63 +482,106 @@ def _patch_class(cls):@functools.wraps(forward)def forward_keep_adapters(self, hidden_states, *args, **kwargs):- if self.training:- if "cache_params" in kwargs:- cache_params = kwargs["cache_params"]+ leaves = mamba_raw_weight_adapters(self)+ if not leaves:+ return forward(self, hidden_states, *args, **kwargs)+ if "cache_params" in kwargs:+ cache_params = kwargs["cache_params"]+ else:+ cache_params = args[cache_index] if 0 <= cache_index < len(args) else None+ compiling = torch.compiler.is_compiling()+ readable = [leaf for leaf in leaves if _install_raw_read(self, leaf)]+ raw, depth = _routing.raw, len(_routing.raw)+ if not (self.training and cache_params is None):+ # Eval / cached decode: the torch path reads dt_proj.weight raw; serve W + delta lazily.+ batch = hidden_states.shape[0] if isinstance(hidden_states, torch.Tensor) else None+ for leaf in readable:+ raw.append((self._modules[leaf], None, f"{cls.__name__}.{leaf}", batch))+ try:+ return forward(self, hidden_states, *args, **kwargs)+ finally:+ while len(raw) > depth:+ raw.pop()+ names = _gate_module_fns(cls)+ modules = {leaf: self._modules[leaf] for leaf in leaves}+ weights = {leaf: _merged_weight(modules[leaf]) for leaf in leaves} if names else {}+ fused_cover = [leaf for leaf in leaves if weights.get(leaf) is not None]+ # Leave the fused kernel only when the non-fused path applies more adapters.+ use_slow = sum(leaf in called for leaf in leaves) > len(fused_cover)+ merged = []+ frame = _Frame(merged)+ for leaf in fused_cover:+ module = modules[leaf]+ if leaf in readable:+ # `.weight` now returns this tensor, so the torch path's raw read applies it too.+ raw.append((module, weights[leaf], f"{cls.__name__}.{leaf}", None))+ frame.applied.add(leaf)+ if not use_slow:+ base = module.get_base_layer().weight+ merged.append((base, weights[leaf], leaf))+ merged.append((weights[leaf], weights[leaf], leaf))+ hooks = []+ if not compiling:+ hooks = [+ module.register_forward_pre_hook(+ functools.partial(_mark_applied, frame.applied, leaf)+ )+ for leaf, module in modules.items()+ if leaf not in frame.applied+ ]+ _routing.stack.append(frame)+ try:+ if use_slow:+ out = non_fused(self, hidden_states, *args, **kwargs)else:- cache_params = args[cache_index] if 0 <= cache_index < len(args) else None- leaves = mamba_raw_weight_adapters(self) if cache_params is None else None- if leaves:- names = _gate_module_fns(cls)- modules = {leaf: self._modules[leaf] for leaf in leaves}- weights = {leaf: _merged_weight(modules[leaf]) for leaf in leaves} if names else {}- fused_cover = [leaf for leaf in leaves if weights.get(leaf) is not None]- # Leave the fused kernel only when the non-fused path applies more adapters.- use_slow = sum(leaf in called for leaf in leaves) > len(fused_cover)- merged = {}- if not use_slow:- for leaf in fused_cover:- base = modules[leaf].get_base_layer().weight- merged[id(base)] = (base, weights[leaf], leaf)- frame = _Frame(merged)- hooks = [- module.register_forward_pre_hook(- functools.partial(_mark_applied, frame.applied, leaf)- )- for leaf, module in modules.items()- ]- _routing.stack.append(frame)- try:- if use_slow:- out = non_fused(self, hidden_states, *args, **kwargs)- else:- # Fused kernel kept; it receives W + delta in place of each adapted W.- out = forward(self, hidden_states, *args, **kwargs)- finally:- _routing.stack.pop()- for hook in hooks:- hook.remove()- _warn_not_applied(cls, [leaf for leaf in leaves if leaf not in frame.applied])- return out- return forward(self, hidden_states, *args, **kwargs)+ # Fused kernel kept; it receives W + delta in place of each adapted W.+ out = forward(self, hidden_states, *args, **kwargs)+ finally:+ _routing.stack.pop()+ for hook in hooks:+ hook.remove()+ while len(raw) > depth:+ raw.pop()+ if not compiling:+ _warn_not_applied(cls, [leaf for leaf in leaves if leaf not in frame.applied])+ return outsetattr(forward_keep_adapters, _MARK, True)cls.forward = forward_keep_adapters+ _prepare_on_assign(cls)return True+def _prepare_on_assign(cls):+ # PEFT attaches adapters via setattr on the mixer: install the class there, before any compile.+ original = cls.__setattr__+ if getattr(original, _MARK, False):+ return++ @functools.wraps(original)+ def __setattr__(self, name, value):+ original(self, name, value)+ if name in _RAW_WEIGHT_LEAVES and not torch.compiler.is_compiling():+ _prepare_leaves(self)++ setattr(__setattr__, _MARK, True)+ cls.__setattr__ = __setattr__++def patch_mamba_fused_lora(model):"""Keep LoRA on Mamba mixers whose fused kernels read x_proj / dt_proj / out_proj raw."""if not isinstance(model, torch.nn.Module):return model- seen = set()+ patched = {}for module in model.modules():cls = type(module)- if cls in seen or not _is_mamba_mixer(module):+ if not _is_mamba_mixer(module):continue- seen.add(cls)- try:- _patch_class(cls)- except Exception:- pass+ if cls not in patched:+ try:+ patched[cls] = _patch_class(cls)+ except Exception:+ patched[cls] = False+ if patched[cls]:+ _prepare_leaves(module)return model