Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Pass layers_pattern from FastLanguageModel.get_peft_model to LoraConfig (#13751)

Vineeth Sai Varikuntlae59b52eb9dad
diff --git a/tests/test_peft_layers_pattern.py b/tests/test_peft_layers_pattern.py
new file mode 100644
index 00000000000..cb0d7f39c02
--- /dev/null
+++ b/tests/test_peft_layers_pattern.py
@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Without layers_pattern PEFT takes the first `<name>.<int>.` as the layer index, so other numbered stacks match too."""
+
+import pytest
+import torch
+
+import unsloth # noqa: F401
+import unsloth.models.llama as llama_module
+from peft import get_peft_model
+from unsloth.models.llama import FastLlamaModel
+
+
+def _tiny_llama_with_aux_stack():
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ config = LlamaConfig(
+ vocab_size = 64,
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_hidden_layers = 2,
+ num_attention_heads = 2,
+ num_key_value_heads = 2,
+ )
+ model = LlamaForCausalLM(config)
+ model.max_seq_length = 64
+ model._saved_temp_tokenizer = None
+ # A second numbered stack that owns a q_proj, like a vision tower or a per-expert list.
+ model.model.aux = torch.nn.ModuleList(
+ [torch.nn.ModuleDict({"q_proj": torch.nn.Linear(16, 16)}) for _ in range(2)]
+ )
+ return model
+
+
+class _Captured(Exception):
+ pass
+
+
+def _lora_config(monkeypatch, **kwargs):
+ def capture(model, lora_config, *args, **kw):
+ raise _Captured(lora_config)
+
+ monkeypatch.setattr(llama_module, "_get_peft_model", capture)
+ # An earlier FastModel.from_pretrained in this process can leave either at 1, skipping LoraConfig.
+ monkeypatch.setenv("UNSLOTH_USE_NEW_MODEL", "0")
+ monkeypatch.setenv("UNSLOTH_ENABLE_FULL_FINETUNING", "0")
+ with pytest.raises(_Captured) as info:
+ FastLlamaModel.get_peft_model(
+ _tiny_llama_with_aux_stack(),
+ r = 4,
+ target_modules = ["q_proj"],
+ use_gradient_checkpointing = False,
+ **kwargs,
+ )
+ return info.value.args[0]
+
+
+def _lora_modules(lora_config):
+ wrapped = get_peft_model(_tiny_llama_with_aux_stack(), lora_config)
+ return sorted(
+ name.removeprefix("base_model.model.").removesuffix(".lora_A")
+ for name, _ in wrapped.named_modules()
+ if name.endswith(".lora_A")
+ )
+
+
+def test_layers_pattern_reaches_lora_config(monkeypatch):
+ config = _lora_config(monkeypatch, layers_to_transform = [1], layers_pattern = "layers")
+ assert config.layers_pattern == "layers"
+
+
+def test_layers_pattern_keeps_the_aux_stack_out(monkeypatch):
+ config = _lora_config(monkeypatch, layers_to_transform = [1], layers_pattern = "layers")
+ assert _lora_modules(config) == ["model.layers.1.self_attn.q_proj"]
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 0d93e3a8cbd..a4da695e551 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -4373,6 +4373,7 @@ class FastLlamaModel:
bias = bias,
task_type = TaskType.CAUSAL_LM if not is_classification else TaskType.SEQ_CLS,
layers_to_transform = layers_to_transform,
+ layers_pattern = layers_pattern,
init_lora_weights = init_lora_weights,
loftq_config = loftq_config,
use_rslora = use_rslora,