Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Hand a composition with no forward of its own to its thinker (Qwen3-Omni) (#11523)
Daniel Han3395708dbd15
diff --git a/tests/test_omni_text_only_load.py b/tests/test_omni_text_only_load.pynew file mode 100644index 00000000000..0dd16cf98ce--- /dev/null+++ b/tests/test_omni_text_only_load.py@@ -0,0 +1,603 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Qwen3-Omni loads with text_only = True, and a kept wrapper still takes a text forward.++Qwen3OmniMoeConfig has no top-level vision_config (it lives under thinker_config) and no+causal-LM class, so text_only = True used to send it to AutoModelForCausalLM, which raised+"Unrecognized configuration class". A default load kept the wrapper, whose missing forward+made model(input_ids = ...) reach nn.Module.forward.+"""++import pytest+import torch+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++transformers = pytest.importorskip("transformers")+omni_config = pytest.importorskip(+ "transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe",+ reason = "this transformers has no qwen3_omni_moe",+)+omni_modeling = pytest.importorskip("transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe")++VOCAB = 64+# Unsloth's patched kernels are Triton on a GPU host, so the tiny model runs where they do.+DEVICE = "cuda" if has_real_cuda() else "cpu"+++def _tiny_config():+ text = dict(+ vocab_size = VOCAB,+ hidden_size = 16,+ intermediate_size = 32,+ moe_intermediate_size = 8,+ shared_expert_intermediate_size = 8,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 8,+ num_experts = 2,+ num_experts_per_tok = 1,+ max_position_embeddings = 64,+ rope_scaling = {"rope_type": "default", "mrope_section": [1, 1, 2], "interleaved": True},+ )+ config = omni_config.Qwen3OmniMoeConfig(+ enable_audio_output = True,+ thinker_config = dict(+ text_config = text,+ # Special ids above anything the tests feed in, so a text prompt stays text.+ audio_start_token_id = 56,+ audio_end_token_id = 57,+ audio_token_id = 58,+ image_token_id = 59,+ video_token_id = 60,+ vision_start_token_id = 61,+ vision_end_token_id = 62,+ audio_config = dict(+ num_mel_bins = 8,+ encoder_layers = 1,+ encoder_attention_heads = 2,+ encoder_ffn_dim = 16,+ d_model = 16,+ output_dim = 16,+ n_window = 2,+ n_window_infer = 4,+ downsample_hidden_size = 8,+ conv_chunksize = 10,+ max_source_positions = 16,+ ),+ vision_config = dict(+ depth = 1,+ hidden_size = 16,+ intermediate_size = 16,+ num_heads = 2,+ out_hidden_size = 16,+ patch_size = 2,+ spatial_merge_size = 1,+ temporal_patch_size = 1,+ deepstack_visual_indexes = [0],+ ),+ ),+ talker_config = dict(+ text_config = text,+ code_predictor_config = dict(+ vocab_size = 16,+ hidden_size = 16,+ intermediate_size = 16,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 8,+ num_code_groups = 2,+ ),+ thinker_hidden_size = 16,+ num_code_groups = 2,+ spatial_merge_size = 1,+ ),+ code2wav_config = dict(+ hidden_size = 16,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ intermediate_size = 16,+ codebook_size = 16,+ decoder_dim = 16,+ num_quantizers = 2,+ upsample_rates = (2,),+ upsampling_ratios = (2,),+ head_dim = 8,+ sliding_window = 8,+ ),+ )+ config.architectures = ["Qwen3OmniMoeForConditionalGeneration"]+ # transformers 5.4's top-level config does not declare the field its _init_weights reads.+ if getattr(config, "initializer_range", None) is None:+ config.initializer_range = 0.02+ return config+++def _tiny_omni():+ torch.manual_seed(0)+ try:+ return omni_modeling.Qwen3OmniMoeForConditionalGeneration(_tiny_config()).to(DEVICE).eval()+ except Exception as error: # a transformers whose Omni config takes other names+ pytest.skip(f"cannot build a tiny Qwen3-Omni here: {error}")+++def _capture_fast_base_kwargs(monkeypatch, tmp_path, text_only):+ from unsloth import FastModel+ from unsloth.models.vision import FastBaseModel++ _tiny_config().save_pretrained(tmp_path)+ seen = {}++ def capture(*args, **kwargs):+ seen.update(kwargs)+ raise RuntimeError("captured")++ monkeypatch.setattr(FastBaseModel, "from_pretrained", staticmethod(capture))+ try:+ FastModel.from_pretrained(str(tmp_path), text_only = text_only, load_in_4bit = False)+ except RuntimeError as error:+ if str(error) != "captured":+ if not has_real_cuda():+ pytest.skip(f"FastModel.from_pretrained needs a GPU here: {error}")+ raise+ else:+ pytest.fail("FastModel.from_pretrained did not reach FastBaseModel.from_pretrained")+ return seen+++def test_text_only_loads_qwen3_omni_through_its_own_auto_class(monkeypatch, tmp_path):+ from transformers import AutoModelForCausalLM+ from unsloth.models._utils import resolve_model_class++ seen = _capture_fast_base_kwargs(monkeypatch, tmp_path, text_only = True)+ auto_model = seen["auto_model"]+ assert auto_model is not AutoModelForCausalLM+ assert resolve_model_class(auto_model, _tiny_config()) is not None+ # The full composition loads; the caller's text intent hands it to the thinker.+ assert seen["text_only"] is False+ assert seen["text_intent"] is True+ assert seen["text_only_decoder"] is False+++def test_the_default_load_resolves_the_same_class(monkeypatch, tmp_path):+ default = _capture_fast_base_kwargs(monkeypatch, tmp_path, text_only = False)+ text = _capture_fast_base_kwargs(monkeypatch, tmp_path, text_only = True)+ assert default["auto_model"] is text["auto_model"]+ assert default["text_intent"] is False+++def test_text_intent_hands_qwen3_omni_to_its_thinker(monkeypatch):+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")+ from unsloth.models.vision import _text_trainable_core++ model = _tiny_omni()+ core = _text_trainable_core(model, text_intent = True)+ assert type(core).__name__ == "Qwen3OmniMoeThinkerForConditionalGeneration"+ # generate and save read the architecture off the core; a sub-config names none.+ assert core.config.architectures == ["Qwen3OmniMoeThinkerForConditionalGeneration"]+ ids = torch.tensor([[1, 2, 3, 4]], device = DEVICE)+ out = core(input_ids = ids, labels = ids)+ assert out.logits.shape == (1, 4, VOCAB)+ assert torch.isfinite(out.loss)+ generated = core.generate(input_ids = ids, max_new_tokens = 2, do_sample = False)+ assert generated.shape == (1, 6)+++def test_a_kept_qwen3_omni_wrapper_forwards_through_its_thinker(capsys, monkeypatch):+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")+ from unsloth.models.vision import _text_trainable_core++ model = _tiny_omni()+ kept = _text_trainable_core(model, text_intent = False)+ assert kept is model+ # Nothing dropped: the talker still generates audio.+ assert hasattr(model, "talker") and hasattr(model, "code2wav")+ assert "text_only = True" in capsys.readouterr().out+ ids = torch.tensor([[1, 2, 3, 4]], device = DEVICE)+ with torch.no_grad():+ out = model(input_ids = ids, labels = ids)+ reference = model.thinker(input_ids = ids, labels = ids)+ assert torch.equal(out.logits, reference.logits)+ assert torch.isfinite(out.loss)+ assert model.get_output_embeddings() is model.thinker.lm_head+ assert model.get_input_embeddings() is model.thinker.get_input_embeddings()+ text = model.generate(+ input_ids = ids, return_audio = False, thinker_max_new_tokens = 2, do_sample = False+ )+ if isinstance(text, tuple): # transformers 4.x returns (sequences, None) without audio+ text = text[0]+ assert text.shape == (1, 6)+ # deepcopy (and so a reference model for KD or DPO) follows the copy's own thinker.+ import copy++ clone = copy.deepcopy(model)+ assert clone.get_output_embeddings() is clone.thinker.lm_head+ assert clone.forward.__self__ is clone.thinker+ # Only this instance changed; the class still has no forward.+ assert type(model).forward is torch.nn.Module.forward+ fresh = _tiny_omni()+ with pytest.raises(TypeError, match = "input_ids"):+ fresh(input_ids = ids)+++def test_a_kept_wrapper_trains_through_peft():+ peft = pytest.importorskip("peft")+ from unsloth.models.vision import _text_trainable_core++ model = _text_trainable_core(_tiny_omni(), text_intent = False)+ # Reentrant checkpointing with frozen embeddings trains LoRA only when the embedding+ # output requires grad, which goes through the wrapper's get_input_embeddings.+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})+ model.enable_input_require_grads()+ config = peft.LoraConfig(+ r = 2, target_modules = r"thinker\.model\.layers\.\d+\.self_attn\.q_proj", lora_alpha = 4+ )+ peft_model = peft.get_peft_model(model, config)+ peft_model.train()+ ids = torch.tensor([[1, 2, 3, 4]], device = DEVICE)+ for name, parameter in peft_model.named_parameters():+ if "lora_B" in name:+ torch.nn.init.normal_(parameter, std = 0.1)+ loss = peft_model(input_ids = ids, labels = ids).loss+ loss.backward()+ grads = [p.grad for n, p in peft_model.named_parameters() if "lora_A" in n]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)+ assert peft_model.get_output_embeddings() is model.thinker.lm_head+++def test_the_off_switch_leaves_the_wrapper_without_a_forward(monkeypatch):+ from unsloth.models.vision import _text_trainable_core++ monkeypatch.setenv("UNSLOTH_KEEP_COMPOSED_WRAPPER", "1")+ model = _tiny_omni()+ assert _text_trainable_core(model, text_intent = False) is model+ assert "forward" not in vars(model)+++def test_a_kept_wrapper_scopes_lora_to_the_thinker_decoder():+ """get_peft_model's regex must reach thinker.model.layers, not only the vision tower."""+ import inspect+ import re+ from unsloth.models.vision import _text_core_decoder_prefix, _text_trainable_core+ from unsloth_zoo.peft_utils import get_peft_regex++ if "language_tags" not in inspect.signature(get_peft_regex).parameters:+ pytest.skip("this unsloth_zoo's get_peft_regex takes no language_tags")+ model = _text_trainable_core(_tiny_omni(), text_intent = False)+ assert _text_core_decoder_prefix(model) == "thinker.model"+ default_tags = inspect.signature(get_peft_regex).parameters["language_tags"].default+ regex = get_peft_regex(model, language_tags = list(default_tags) + [re.escape("thinker.model")])+ targets = [name for name, _ in model.named_modules() if re.fullmatch(regex, name)]+ assert any(name.startswith("thinker.model.layers.") for name in targets)+ assert not any(+ name.startswith(("talker.", "code2wav.", "thinker.audio_tower.")) for name in targets+ )+ # Without the core's prefix only the vision tower matched, so a text batch trained nothing.+ before = get_peft_regex(model)+ assert not any(+ name.startswith("thinker.model.layers.")+ for name, _ in model.named_modules()+ if re.fullmatch(before, name)+ )+ assert _text_core_decoder_prefix(_tiny_omni()) is None+++def test_get_peft_model_passes_the_thinker_decoder_as_a_language_tag(monkeypatch):+ import inspect+ from unsloth.models import vision++ if "language_tags" not in inspect.signature(vision.get_peft_regex).parameters:+ pytest.skip("this unsloth_zoo's get_peft_regex takes no language_tags")+ import functools++ seen = {}++ # functools.wraps keeps the real signature, which get_peft_model inspects.+ @functools.wraps(vision.get_peft_regex)+ def capture(model, **kwargs):+ seen.update(kwargs)+ raise RuntimeError("captured")++ monkeypatch.setattr(vision, "get_peft_regex", capture)+ model = vision._text_trainable_core(_tiny_omni(), text_intent = False)+ with pytest.raises(RuntimeError, match = "captured"):+ vision.FastBaseModel.get_peft_model(model, r = 2)+ assert "thinker\\.model" in seen["language_tags"]+ seen.clear()+ thinker = vision._text_trainable_core(_tiny_omni(), text_intent = True)+ with pytest.raises(RuntimeError, match = "captured"):+ vision.FastBaseModel.get_peft_model(thinker, r = 2)+ assert "language_tags" not in seen+++def test_a_kept_wrapper_with_an_accelerate_hook_forwards_through_the_hook(monkeypatch):+ """A device-mapped load wraps forward on the instance; the hook must reach the thinker."""+ hooks = pytest.importorskip("accelerate.hooks")+ from unsloth.models.vision import _text_trainable_core++ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")+ calls = []++ class Counting(hooks.ModelHook):+ def pre_forward(self, module, *args, **kwargs):+ calls.append(type(module).__name__)+ return args, kwargs++ model = _tiny_omni()+ hooks.add_hook_to_module(model, Counting())+ model = _text_trainable_core(model, text_intent = False)+ ids = torch.tensor([[1, 2, 3, 4]], device = DEVICE)+ with torch.no_grad():+ out = model(input_ids = ids, labels = ids)+ assert torch.isfinite(out.loss)+ assert calls == ["Qwen3OmniMoeForConditionalGeneration"]+++def _capture_adapter_reload(monkeypatch, tmp_path, target_modules):+ peft = pytest.importorskip("peft")+ from unsloth import FastModel+ from unsloth.models.vision import FastBaseModel++ base = tmp_path / "base"+ adapter = tmp_path / "adapter"+ _tiny_config().save_pretrained(base)+ peft.LoraConfig(+ r = 2, lora_alpha = 2, target_modules = target_modules, base_model_name_or_path = str(base)+ ).save_pretrained(adapter)+ seen = {}++ def capture(*args, **kwargs):+ seen.update(kwargs)+ raise RuntimeError("captured")++ monkeypatch.setattr(FastBaseModel, "from_pretrained", staticmethod(capture))+ try:+ FastModel.from_pretrained(str(adapter), load_in_4bit = False)+ except RuntimeError as error:+ if str(error) != "captured":+ if not has_real_cuda():+ pytest.skip(f"FastModel.from_pretrained needs a GPU here: {error}")+ raise+ return seen+++def test_an_adapter_trained_on_the_thinker_reloads_onto_the_thinker(monkeypatch, tmp_path):+ # A text_only adapter's regex is rooted at model.layers; the composition names them+ # thinker.model.layers, so a default reload could not find its targets.+ seen = _capture_adapter_reload(+ monkeypatch, tmp_path, r"(?:\bmodel\.layers\.[\d]{1,}\.(?:self_attn)\.(?:q_proj))"+ )+ assert seen["text_intent"] is True+++def test_an_adapter_trained_on_the_kept_wrapper_keeps_the_wrapper(monkeypatch, tmp_path):+ seen = _capture_adapter_reload(+ monkeypatch, tmp_path, r"(?:.*?(?:thinker\.model).*?(?:self_attn).*?(?:q_proj))"+ )+ assert seen["text_intent"] is False+++def _tiny_omni_checkpoint(path):+ tokenizers = pytest.importorskip("tokenizers")+ _tiny_omni().save_pretrained(path)+ vocab = {f"w{i}": i for i in range(48)}+ vocab.update({"<unk>": 48, "<pad>": 49, "<eos>": 50})+ backend = tokenizers.Tokenizer(tokenizers.models.WordLevel(vocab, unk_token = "<unk>"))+ backend.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace()+ transformers.PreTrainedTokenizerFast(+ tokenizer_object = backend, unk_token = "<unk>", pad_token = "<pad>", eos_token = "<eos>"+ ).save_pretrained(path)++[email protected]("text_only", [True, False], ids = ["thinker", "kept_wrapper"])+ "target_modules", [None, ["q_proj", "v_proj"]], ids = ["unsloth_regex", "leaf_list"]+)+def test_an_omni_adapter_reloads_onto_the_model_it_was_trained_on(+ tmp_path, text_only, target_modules+):+ # A leaf-name list reloads as a set that matches either layout, so the saved weight+ # keys (model.layers vs thinker.model.layers) are what decide.+ if not has_real_cuda():+ pytest.skip("FastModel needs a GPU")+ from unsloth import FastModel++ base = tmp_path / "base"+ _tiny_omni_checkpoint(base)+ model, _ = FastModel.from_pretrained(str(base), text_only = text_only, load_in_4bit = False)+ kwargs = {} if target_modules is None else {"target_modules": target_modules}+ model = FastModel.get_peft_model(model, r = 2, lora_alpha = 2, random_state = 0, **kwargs)+ trained_on = type(model.get_base_model()).__name__+ with torch.no_grad():+ for name, param in model.named_parameters():+ if "lora_B" in name:+ param.normal_(0, 0.1)+ saved = {+ n: p.detach().float().cpu().clone() for n, p in model.named_parameters() if "lora_" in n+ }+ assert saved+ model.save_pretrained(tmp_path / "adapter")+ del model+ reloaded, _ = FastModel.from_pretrained(str(tmp_path / "adapter"), load_in_4bit = False)+ assert type(reloaded.get_base_model()).__name__ == trained_on+ loaded = {n: p.detach().float().cpu() for n, p in reloaded.named_parameters() if "lora_" in n}+ assert loaded.keys() == saved.keys()+ for name, value in saved.items():+ assert torch.equal(loaded[name], value), name+++def test_saved_weight_keys_decide_over_the_target_regex():+ from unsloth.models.loader import _adapter_targets_text_core++ config = type("Config", (), {"target_modules": {"q_proj", "v_proj"}})()+ thinker_keys = ["base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight"]+ wrapper_keys = ["base_model.model.thinker.model.layers.0.self_attn.q_proj.lora_A.weight"]+ assert _adapter_targets_text_core(config, thinker_keys) is True+ assert _adapter_targets_text_core(config, wrapper_keys) is False+ # Without keys a leaf list cannot tell the layouts apart, so the composition is kept.+ assert _adapter_targets_text_core(config, None) is False+ regex = type("Config", (), {"target_modules": r".*\.q_proj"})()+ assert _adapter_targets_text_core(regex, wrapper_keys) is False++[email protected]("saved_as", ["adapter_model.safetensors", "adapter_model.bin"])+def test_hub_adapter_keys_are_read_from_the_saved_file(monkeypatch, tmp_path, saved_as):+ # Resolved through hf_hub_download (cache first, offline honoured) in either format.+ huggingface_hub = pytest.importorskip("huggingface_hub")+ safetensors_torch = pytest.importorskip("safetensors.torch")+ from unsloth.models.loader import _adapter_weight_keys++ key = "base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight"+ path = tmp_path / saved_as+ if saved_as.endswith(".safetensors"):+ safetensors_torch.save_file({key: torch.ones(2, 2)}, str(path))+ else:+ torch.save({key: torch.ones(2, 2)}, path)+ asked = []++ def fake_download(repo_id, filename, **kwargs):+ asked.append((repo_id, filename, kwargs.get("local_files_only"), kwargs.get("revision")))+ if filename != saved_as:+ raise FileNotFoundError(filename)+ return str(path)++ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)+ assert _adapter_weight_keys("someone/omni-adapter", revision = "r", local_files_only = True) == [+ key+ ]+ assert asked[0] == ("someone/omni-adapter", "adapter_model.safetensors", True, "r")+ assert asked[-1][1] == saved_as+++def test_local_bin_adapter_keys_are_read_without_weights(tmp_path):+ from unsloth.models.loader import _adapter_weight_keys++ key = "base_model.model.thinker.model.layers.0.q_proj.lora_A.weight"+ torch.save({key: torch.ones(2, 2)}, tmp_path / "adapter_model.bin")+ assert _adapter_weight_keys(str(tmp_path)) == [key]+ assert _adapter_weight_keys(str(tmp_path / "missing")) is None+++def test_an_audio_only_wrapper_adapter_stays_on_the_wrapper():+ # Trained on the kept wrapper's talker only: no thinker. key, but still wrapper-rooted.+ from unsloth.models.loader import _adapter_targets_text_core, _composition_children++ children = _composition_children(_tiny_config())+ assert set(children) == {"thinker", "talker", "code2wav"}+ config = type("Config", (), {"target_modules": {"q_proj"}})()+ talker = ["base_model.model.talker.model.layers.0.self_attn.q_proj.lora_A.weight"]+ code2wav = ["base_model.model.code2wav.decoder.0.conv.lora_A.weight"]+ thinker_only = ["base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight"]+ assert _adapter_targets_text_core(config, talker, children) is False+ assert _adapter_targets_text_core(config, code2wav, children) is False+ assert _adapter_targets_text_core(config, thinker_only, children) is True+ regex = type("Config", (), {"target_modules": r"(?:.*?(?:talker).*?(?:q_proj))"})()+ assert _adapter_targets_text_core(regex, None, children) is False+++def test_only_a_forwardless_composition_redirects_adapter_reloads():+ """Every VLM maps to an image-text auto class; routing their adapters to text_only dropped the vision tower."""+ from transformers import Gemma3Config, Qwen2_5_VLConfig+ from unsloth.models.loader import _is_forwardless_composition++ assert _is_forwardless_composition(_tiny_config()) is True+ assert _is_forwardless_composition(Gemma3Config()) is False+ assert _is_forwardless_composition(Qwen2_5_VLConfig()) is False+++def test_a_kept_wrapper_freezes_the_siblings_its_forward_never_uses():+ """DDP (find_unused_parameters=False) fails on trainable talker / code2wav weights with no gradient."""+ from unsloth.models.vision import _text_trainable_core++ model = _text_trainable_core(_tiny_omni(), text_intent = False)+ assert all(not p.requires_grad for p in model.talker.parameters())+ assert all(not p.requires_grad for p in model.code2wav.parameters())+ assert any(p.requires_grad for p in model.thinker.parameters())+++def test_a_kept_wrapper_resolves_its_layer_count_through_the_thinker():+ from unsloth.models.vision import _get_total_transformer_layers, _text_trainable_core++ model = _text_trainable_core(_tiny_omni(), text_intent = False)+ assert _get_total_transformer_layers(model) is None+ assert _get_total_transformer_layers(model.thinker) == model.thinker.config.text_config.num_hidden_layers+++def test_an_explicit_leaf_list_on_a_kept_wrapper_is_scoped_to_the_thinker(monkeypatch):+ """PEFT suffix-matches ["q_proj", "v_proj"] in the talker too; those adapters never train."""+ import functools+ from unsloth.models import vision++ seen = {}++ @functools.wraps(vision.get_peft_regex)+ def capture(model, **kwargs):+ seen.update(kwargs)+ raise RuntimeError("captured")++ monkeypatch.setattr(vision, "get_peft_regex", capture)+ model = vision._text_trainable_core(_tiny_omni(), text_intent = False)+ with pytest.raises(RuntimeError, match = "captured"):+ vision.FastBaseModel.get_peft_model(model, r = 2, target_modules = ["q_proj", "v_proj"])+ assert seen["target_modules"] == ["q_proj", "v_proj"]+++def test_a_kept_wrapper_resizes_its_vocabulary_through_the_thinker():+ from unsloth.models.vision import _text_trainable_core++ model = _text_trainable_core(_tiny_omni(), text_intent = False)+ old = model.get_input_embeddings().weight.shape[0]+ model.resize_token_embeddings(old + 8)+ assert model.thinker.get_input_embeddings().weight.shape[0] == old + 8+ assert model.thinker.lm_head.weight.shape[0] == old + 8+++def _kept_wrapper_peft(monkeypatch, **peft_kwargs):+ from unsloth.models import vision++ monkeypatch.setattr(vision.FastBaseModel, "post_patch_model", staticmethod(lambda model, *a, **k: model))+ model = vision._text_trainable_core(_tiny_omni(), text_intent = False)+ model.max_seq_length = 64+ return vision.FastBaseModel.get_peft_model(model, r = 2, **peft_kwargs)+++def test_a_qualified_target_on_a_kept_wrapper_reaches_peft_unchanged(monkeypatch):+ """get_peft_regex treats entries as leaves, so a full path raised "No layers to finetune"."""+ target = "thinker.model.layers.0.self_attn.q_proj"+ peft_model = _kept_wrapper_peft(monkeypatch, target_modules = [target])+ trained = {n for n, p in peft_model.named_parameters() if p.requires_grad}+ assert trained and all(target in n for n in trained)+++def test_saved_modules_on_a_kept_wrapper_skip_the_talker(monkeypatch):+ """An unqualified "lm_head" also suffix-matched talker.code_predictor.lm_head, a ModuleList PEFT refuses."""+ peft_model = _kept_wrapper_peft(monkeypatch, target_modules = ["q_proj"], modules_to_save = ["lm_head"])+ trained = [n for n, p in peft_model.named_parameters() if p.requires_grad]+ assert any("thinker.lm_head" in n for n in trained)+ assert not any(".talker." in n for n in trained)+++def test_full_finetuning_keeps_a_kept_wrappers_siblings_frozen(monkeypatch):+ """prepare_model_for_training(full_finetuning = True) turned every parameter back on."""+ from unsloth.models import vision++ monkeypatch.setenv("UNSLOTH_ENABLE_FULL_FINETUNING", "1")+ model = vision._text_trainable_core(_tiny_omni(), text_intent = False)+ try:+ vision.FastBaseModel.post_patch_model(model, use_gradient_checkpointing = False)+ except RuntimeError as error:+ if "inner_training_loop" not in str(error):+ raise+ assert any(p.requires_grad for p in model.thinker.parameters())+ assert not any(p.requires_grad for p in model.talker.parameters())+++def test_suffix_target_parameters_on_a_kept_wrapper_skip_the_talker(monkeypatch):+ """PEFT suffix-matches target_parameters; bare expert names also adapted the talker's experts."""+ peft_model = _kept_wrapper_peft(+ monkeypatch, target_modules = ["q_proj"], target_parameters = ["mlp.experts.gate_up_proj"]+ )+ trained = [n for n, p in peft_model.named_parameters() if p.requires_grad]+ assert any("thinker." in n and "experts" in n for n in trained)+ assert not any(".talker." in n for n in trained)diff --git a/tests/test_omni_trains_through_thinker.py b/tests/test_omni_trains_through_thinker.pynew file mode 100644index 00000000000..dcf3a65295d--- /dev/null+++ b/tests/test_omni_trains_through_thinker.py@@ -0,0 +1,192 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""A composition with no forward of its own (Qwen3-Omni) trains through its thinker."""++import pytest+import torch++transformers = pytest.importorskip("transformers")+from transformers import PreTrainedModel, PretrainedConfig+++class TinyConfig(PretrainedConfig):+ model_type = "tiny_composed"++ def __init__(+ self,+ vocab_size = 16,+ hidden_size = 8,+ **kwargs,+ ):+ self.vocab_size = vocab_size+ self.hidden_size = hidden_size+ super().__init__(**kwargs)+++class Thinker(PreTrainedModel):+ config_class = TinyConfig++ def __init__(self, config):+ super().__init__(config)+ self.embed_tokens = torch.nn.Embedding(config.vocab_size, config.hidden_size)+ self.lm_head = torch.nn.Linear(config.hidden_size, config.vocab_size, bias = False)++ def get_input_embeddings(self):+ return self.embed_tokens++ def get_output_embeddings(self):+ return self.lm_head++ def forward(+ self,+ input_ids = None,+ **kwargs,+ ):+ return self.lm_head(self.embed_tokens(input_ids))+++class Talker(PreTrainedModel):+ config_class = TinyConfig++ def __init__(self, config):+ super().__init__(config)+ self.proj = torch.nn.Linear(config.hidden_size, config.hidden_size)++ def forward(+ self,+ hidden_states = None,+ **kwargs,+ ):+ return self.proj(hidden_states)+++class Composed(PreTrainedModel):+ """No forward of its own, like Qwen3OmniMoeForConditionalGeneration."""++ config_class = TinyConfig++ def __init__(self, config):+ super().__init__(config)+ self.thinker = Thinker(config)+ self.talker = Talker(config)+ self.code2wav = torch.nn.Linear(config.hidden_size, 1)+++def test_the_defect_before_the_fix():+ """The arm that fails on main."""+ model = Composed(TinyConfig())+ with pytest.raises(TypeError, match = "unexpected keyword argument 'input_ids'"):+ model(input_ids = torch.tensor([[1, 2]]))+++def test_composition_without_forward_hands_off_to_the_thinker():+ from unsloth.models.vision import _text_trainable_core++ model = Composed(TinyConfig())+ core = _text_trainable_core(model)+ assert isinstance(core, Thinker)+ assert core._unsloth_composed_parent == "Composed"+ assert not hasattr(model, "talker") and not hasattr(model, "code2wav")+ logits = core(input_ids = torch.tensor([[1, 2, 3]]))+ assert logits.shape == (1, 3, 16)+++def test_a_model_with_a_text_forward_is_returned_unchanged():+ from unsloth.models.vision import _text_trainable_core+ model = Thinker(TinyConfig())+ assert _text_trainable_core(model) is model+++def test_ambiguous_compositions_are_left_alone():+ from unsloth.models.vision import _text_trainable_core++ class TwoCores(PreTrainedModel):+ config_class = TinyConfig++ def __init__(self, config):+ super().__init__(config)+ self.encoder = Thinker(config)+ self.decoder = Thinker(config)++ model = TwoCores(TinyConfig())+ assert _text_trainable_core(model) is model+ assert hasattr(model, "encoder") and hasattr(model, "decoder")+++def test_the_off_switch_keeps_the_wrapper(monkeypatch):+ from unsloth.models.vision import _text_trainable_core++ monkeypatch.setenv("UNSLOTH_KEEP_COMPOSED_WRAPPER", "1")+ model = Composed(TinyConfig())+ assert _text_trainable_core(model) is model+++def test_a_multimodal_load_keeps_the_composition_for_generation(capsys):+ """text_intent = False keeps the composition whole and prints the text_only hint."""+ from unsloth.models.vision import _text_trainable_core++ model = Composed(TinyConfig())+ assert _text_trainable_core(model, text_intent = False) is model+ assert hasattr(model, "talker")+ assert "text_only = True" in capsys.readouterr().out+++def test_the_thinker_carries_the_checkpoint_identity():+ """PEFT writes name_or_path into the adapter's base_model_name_or_path; a thinker built+ from thinker_config has none of its own, so it takes the wrapper's."""+ from types import SimpleNamespace+ from unsloth.models.vision import _carry_loader_state_to_core++ core = torch.nn.Module()+ core.config = SimpleNamespace(_name_or_path = "")+ core.name_or_path = ""+ model = torch.nn.Module()+ model.config = SimpleNamespace(_name_or_path = "Qwen/Qwen3-Omni-30B-A3B-Instruct")+ model.name_or_path = "Qwen/Qwen3-Omni-30B-A3B-Instruct"+ model.thinker = core+ _carry_loader_state_to_core(model, core, "thinker")+ assert core.name_or_path == "Qwen/Qwen3-Omni-30B-A3B-Instruct"+ assert core.config._name_or_path == "Qwen/Qwen3-Omni-30B-A3B-Instruct"+++def test_a_thinker_whose_output_accessor_says_none_is_still_found_by_its_lm_head():+ """transformers 4.x returns None from get_output_embeddings unless a class overrides it;+ Qwen3-Omni's thinker does not, but owns an lm_head."""+ from transformers import PreTrainedModel, PretrainedConfig+ from unsloth.models.vision import _text_trainable_core++ class Cfg(PretrainedConfig):+ model_type = "thinker_for_test"++ class Thinker(PreTrainedModel):+ config_class = Cfg++ def __init__(self, config):+ super().__init__(config)+ self.embed = torch.nn.Embedding(8, 4)+ self.lm_head = torch.nn.Linear(4, 8)++ def get_input_embeddings(self):+ return self.embed++ def get_output_embeddings(self):+ return None++ def forward(+ self,+ input_ids = None,+ labels = None,+ **kwargs,+ ):+ return self.lm_head(self.embed(input_ids))++ class Wrapper(PreTrainedModel):+ config_class = Cfg++ def __init__(self, config):+ super().__init__(config)+ self.thinker = Thinker(config)+ self.talker = torch.nn.Linear(2, 2)++ wrapper = Wrapper(Cfg())+ core = _text_trainable_core(wrapper, text_intent = True)+ assert isinstance(core, Thinker)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 0fce2d6152e..8d8c8e2d59e 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -311,6 +311,97 @@ _OMNI_AUTO_CLASS_NAMES = ()+def _adapter_file_keys(path):+ if path.endswith(".safetensors"):+ from safetensors import safe_open+ with safe_open(path, framework = "pt") as handle:+ return list(handle.keys())+ return list(torch.load(path, map_location = "meta", weights_only = True).keys())+++def _adapter_weight_keys(+ adapter_name,+ token = None,+ revision = None,+ local_files_only = False,+ cache_dir = None,+):+ """Tensor names of a saved adapter without loading weights, or None."""+ filenames = ("adapter_model.safetensors", "adapter_model.bin")+ try:+ local = os.path.expanduser(adapter_name)+ if os.path.isdir(local):+ for filename in filenames:+ path = os.path.join(local, filename)+ if os.path.exists(path):+ return _adapter_file_keys(path)+ return None+ from huggingface_hub import hf_hub_download++ for filename in filenames:+ try:+ path = hf_hub_download(+ adapter_name,+ filename,+ revision = revision,+ token = token,+ cache_dir = cache_dir,+ local_files_only = local_files_only,+ )+ except Exception:+ continue+ return _adapter_file_keys(path)+ except Exception:+ pass+ return None+++def _composition_children(model_config):+ names = [+ name[: -len("_config")]+ for name in (getattr(model_config, "sub_configs", None) or {})+ if name.endswith("_config")+ ]+ return tuple(names) or ("thinker",)+++def _adapter_targets_text_core(+ peft_config,+ weight_keys = None,+ wrapper_children = ("thinker",),+):+ """True when an adapter was trained on an extracted thinker (`text_only = True`).++ Saved keys decide (wrapper-trained keys are rooted at a wrapper child); else the saved+ regex. A leaf-name list matches either layout, so it cannot decide alone.+ """+ children = set(wrapper_children)+ if weight_keys:+ roots = {+ (key[len("base_model.model.") :] if key.startswith("base_model.model.") else key).split(+ ".", 1+ )[0]+ for key in weight_keys+ }+ return not (roots & children)+ targets = getattr(peft_config, "target_modules", None)+ return isinstance(targets, str) and not any(child in targets for child in children)+++def _is_forwardless_composition(model_config, trust_remote_code = None, **hub_kwargs):+ # Only a wrapper with no forward (Qwen3-Omni) can have a thinker-trained adapter; VLM adapters never.+ auto_class = _resolve_omni_auto_model(model_config, trust_remote_code = trust_remote_code, **hub_kwargs)+ model_class = (+ resolve_model_class(auto_class, model_config, trust_remote_code = trust_remote_code, **hub_kwargs)+ if auto_class is not None+ else None+ )+ if model_class is None:+ return False+ forward = getattr(model_class, "forward", None)+ return forward is None or forward is torch.nn.Module.forward++def _config_has_native_class(auto_class, config):"""True when transformers itself maps ``type(config)`` in ``auto_class`` (no repo code needed)."""try:@@ -1965,12 +2056,6 @@ class FastModel(FastBaseModel):for _cfg_key, _cfg_val in task_config_attrs.items():set_task_config_attr(model_config, _cfg_key, _cfg_val)- architectures = getattr(model_config, "architectures", None)- if architectures is None:- architectures = []- is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures)- is_vlm = is_vlm or hasattr(model_config, "vision_config")- load_text_only = text_only and auto_model is None# Class probes below fetch remote modeling code exactly as the load will._probe_hub_kwargs = dict(trust_remote_code = trust_remote_code,@@ -1982,6 +2067,40 @@ class FastModel(FastBaseModel):force_download = kwargs.get("force_download", None),proxies = kwargs.get("proxies", None),)+ architectures = getattr(model_config, "architectures", None)+ if architectures is None:+ architectures = []+ is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures)+ is_vlm = is_vlm or hasattr(model_config, "vision_config")+ if (+ is_peft+ and not text_only+ and auto_model is None+ and _is_forwardless_composition(model_config, **_probe_hub_kwargs)+ ):+ _adapter_keys = _adapter_weight_keys(+ old_model_name,+ token = token,+ revision = adapter_revision,+ local_files_only = local_files_only,+ cache_dir = kwargs.get("cache_dir"),+ )+ if _adapter_targets_text_core(+ peft_config, _adapter_keys, _composition_children(model_config)+ ):+ print(+ "Unsloth: this adapter was trained on the thinker alone (`text_only = True`), "+ "so its base is loaded the same way."+ )+ text_only = True+ elif not _adapter_keys and not isinstance(+ getattr(peft_config, "target_modules", None), str+ ):+ print(+ "Unsloth: could not read this adapter's weight names, so the full model is "+ "loaded. If it was trained with `text_only = True`, pass `text_only = True` here too."+ )+ load_text_only = text_only and auto_model is Nonetext_only_decoder = Falseif load_text_only:if hasattr(model_config, "vision_config"):@@ -2006,6 +2125,13 @@ class FastModel(FastBaseModel):is_vlm = False# model_config is no longer the repo's config, so anything rebuilding it from model_name (the device-map planner) sees a different model.text_only_decoder = True+ elif (+ is_vlm+ and resolve_model_class(AutoModelForCausalLM, model_config, **_probe_hub_kwargs) is None+ and _resolve_omni_auto_model(model_config, **_probe_hub_kwargs) is not None+ ):+ # Qwen3-Omni has no causal-LM class; load the composition, text_intent picks the thinker.+ load_text_only = Falseelse:is_vlm = Falsefor _cfg_key, _cfg_val in task_config_attrs.items():diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 11bf9a9d097..0e13f5ad968 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -944,7 +944,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):if do_bfloat16_mixed_precision:dtype = torch.bfloat16- # text_only configs have architectures=None.+ # A core built from a sub-config (Qwen3-Omni's thinker) carries no architectures.architectures = getattr(self.config, "architectures", None) or [type(self).__name__]is_vlm = any(x.endswith(("ForConditionalGeneration", "ForVisionText2Text")) for x in architectures@@ -1364,27 +1364,25 @@ def _required_non_text_inputs(forward):]-def _text_trainable_core(model, text_intent = True):- """Nemotron-3-Nano-Omni's forward requires `pixel_values`: with text_only, train the one text child instead."""- if os.environ.get("UNSLOTH_KEEP_COMPOSED_WRAPPER", "0") == "1":- return model- forward = getattr(type(model), "forward", None)- if forward is None or forward is torch.nn.Module.forward:- return model- required = _required_non_text_inputs(forward)- if not required:- return model- if not text_intent:- print(- f"Unsloth: `{type(model).__name__}.forward` requires {', '.join(required)}, so it "- "trains on multimodal batches only. Pass `text_only = True` to from_pretrained to "- "train its language model on text batches instead."- )- return model+def _output_embeddings_of(module):+ # transformers 4.x returns None unless overridden; Qwen3-Omni's thinker does not override.+ try:+ output_embeddings = module.get_output_embeddings()+ except Exception:+ output_embeddings = None+ if output_embeddings is None:+ output_embeddings = getattr(module, "lm_head", None)+ if not isinstance(output_embeddings, torch.nn.Module):+ output_embeddings = None+ return output_embeddings+++def _find_text_core(model):+ """(name, child) of the one direct child a text batch can run through, else None."""try:from transformers import PreTrainedModelexcept Exception:- return model+ return Nonecores = []for name, child in model.named_children():if not isinstance(child, PreTrainedModel):@@ -1397,7 +1395,7 @@ def _text_trainable_core(model, text_intent = True):try:has_embeddings = (child.get_input_embeddings() is not None- and child.get_output_embeddings() is not None+ and _output_embeddings_of(child) is not None)except Exception:has_embeddings = False@@ -1407,16 +1405,180 @@ def _text_trainable_core(model, text_intent = True):if len(preferred) == 1:cores = preferredif len(cores) != 1:+ return None+ return cores[0]+++def _text_core_output_embeddings(model):+ return _output_embeddings_of(getattr(model, model._unsloth_text_core))+++def _delegate_text_forward(model, name, core):+ """Instance-only (class and generate untouched): forward + embeddings from the core."""+ if "_old_forward" in vars(model):+ # accelerate's hook calls _old_forward.+ model._old_forward = core.forward+ else:+ model.forward = core.forward+ try:+ wrapper_output = model.get_output_embeddings()+ except Exception:+ wrapper_output = None+ if wrapper_output is None:+ # Bound method, not a closure, so deepcopy and pickle follow it.+ model.get_output_embeddings = types.MethodType(_text_core_output_embeddings, model)+ # Gradient checkpointing needs this, else transformers 5 raises and LoRA gets no gradient.+ try:+ wrapper_input = model.get_input_embeddings()+ except Exception:+ wrapper_input = None+ if wrapper_input is None:+ model.get_input_embeddings = core.get_input_embeddings+ # resize_token_embeddings pairs the getters with these setters.+ model.set_input_embeddings = core.set_input_embeddings+ if wrapper_output is None:+ model.set_output_embeddings = core.set_output_embeddings+ model._unsloth_text_core = name+ _freeze_unused_siblings(model)+ return model+++def _freeze_unused_siblings(model):+ """Siblings never reach the loss: freeze them, else DDP full finetuning fails on unused parameters."""+ name = getattr(model, "_unsloth_text_core", None)+ core = getattr(model, name, None) if isinstance(name, str) else None+ if core is None:+ return+ core_parameters = {id(parameter) for parameter in core.parameters()}+ for child in model.children():+ if child is core:+ continue+ for parameter in child.parameters():+ if id(parameter) not in core_parameters:+ parameter.requires_grad_(False)+++def _scope_parameter_targets_to_core(model, target_parameters):+ """PEFT suffix-matches target_parameters too: on a kept wrapper "mlp.experts.gate_up_proj"+ also hits the talker's experts, which never train."""+ name = getattr(model, "_unsloth_text_core", None)+ core = getattr(model, name, None) if isinstance(name, str) else None+ if core is None or not target_parameters or isinstance(target_parameters, str):+ return target_parameters+ scoped = []+ for entry in target_parameters:+ if entry.startswith(name + "."):+ scoped.append(entry)+ continue+ paths = [+ f"{name}.{path}"+ for path, _ in core.named_parameters()+ if path == entry or path.endswith("." + entry)+ ]+ scoped.extend(paths or [entry])+ return scoped+++def _scope_modules_to_save_to_core(model, modules_to_save):+ """PEFT suffix-matches modules_to_save over the whole model: on a kept wrapper an unqualified+ "lm_head" also hits the talker's code_predictor.lm_head, a ModuleList PEFT refuses."""+ name = getattr(model, "_unsloth_text_core", None)+ core = getattr(model, name, None) if isinstance(name, str) else None+ if core is None or not modules_to_save:+ return modules_to_save+ scoped = []+ for entry in modules_to_save:+ paths = [+ f"{name}.{child}"+ for child, _ in core.named_modules()+ if child and (child == entry or child.endswith("." + entry))+ ]+ scoped.extend(paths or [entry])+ return scoped+++def _text_core_decoder_prefix(model):+ name = getattr(model, "_unsloth_text_core", None)+ core = getattr(model, name, None) if isinstance(name, str) else None+ if core is None:+ return None+ try:+ decoder = core.get_decoder()+ except Exception:+ decoder = None+ if isinstance(decoder, torch.nn.Module) and decoder is not core:+ for child_name, module in core.named_modules():+ if module is decoder and child_name:+ return f"{name}.{child_name}"+ # transformers 5.4's thinker returns itself from get_decoder: use the embeddings' owner.+ try:+ embeddings = core.get_input_embeddings()+ except Exception:+ embeddings = None+ if embeddings is not None:+ for child_name, child in core.named_children():+ try:+ # A vision tower raises here on transformers 5.+ if child.get_input_embeddings() is embeddings:+ return f"{name}.{child_name}"+ except Exception:+ continue+ return name+++def _text_trainable_core(model, text_intent = True):+ """With text_intent, the text child to train when the wrapper's forward needs non-text+ inputs (Nemotron-3-Nano-Omni) or has none (Qwen3-Omni); else a hint, and a forward-less+ wrapper gets its core's forward on the instance."""+ if os.environ.get("UNSLOTH_KEEP_COMPOSED_WRAPPER", "0") == "1":return model- name, core = cores[0]+ forward = getattr(type(model), "forward", None)+ has_no_forward = forward is None or forward is torch.nn.Module.forward+ required = [] if has_no_forward else _required_non_text_inputs(forward)+ if not has_no_forward and not required:+ return model+ if not text_intent:+ if has_no_forward:+ found = _find_text_core(model)+ if found is not None:+ _delegate_text_forward(model, *found)+ print(+ f"Unsloth: `{type(model).__name__}` has no forward of its own, so a text "+ f"forward runs through its `{found[0]}` and the rest is kept for generation. "+ "Pass `text_only = True` to from_pretrained to train the "+ f"`{found[0]}` alone and free the rest."+ )+ else:+ print(+ f"Unsloth: `{type(model).__name__}` has no forward of its own, so it cannot be "+ "trained as loaded. Pass `text_only = True` to from_pretrained to train its "+ "thinker on text batches instead."+ )+ else:+ print(+ f"Unsloth: `{type(model).__name__}.forward` requires {', '.join(required)}, so it "+ "trains on multimodal batches only. Pass `text_only = True` to from_pretrained to "+ "train its language model on text batches instead."+ )+ return model+ found = _find_text_core(model)+ if found is None:+ return model+ name, core = founddropped = [child_name for child_name, _ in model.named_children() if child_name != name]for child_name in dropped:delattr(model, child_name)gc.collect()clean_gpu_cache()+ if has_no_forward:+ reason = f"`{type(model).__name__}` has no forward of its own"+ else:+ reason = (+ f"`{type(model).__name__}.forward` requires {', '.join(required)}, which a text "+ "batch does not carry"+ )print(- f"Unsloth: `{type(model).__name__}.forward` requires {', '.join(required)}, which a text "- f"batch does not carry, so training uses its `{name}` (`{type(core).__name__}`)."+ f"Unsloth: {reason}, so training uses its `{name}` (`{type(core).__name__}`)."+ (f" Dropped {', '.join(dropped)}: not used by text training." if dropped else "")+ " Saving writes a checkpoint of that module.")@@ -1458,20 +1620,26 @@ def _carry_loader_state_to_core(model, core, name):carried[key[len(prefix) :]] = deviceif carried:core.hf_device_map = carried+ # PEFT writes name_or_path into base_model_name_or_path; sub-config children lack one.+ wrapper_name = getattr(model, "name_or_path", None) or getattr(+ getattr(model, "config", None), "_name_or_path", None+ )+ if wrapper_name:+ if not getattr(core, "name_or_path", None):+ try:+ core.name_or_path = wrapper_name+ except Exception:+ pass+ _core_cfg = getattr(core, "config", None)+ if _core_cfg is not None and not getattr(_core_cfg, "_name_or_path", None):+ try:+ _core_cfg._name_or_path = wrapper_name+ except Exception:+ passwrapper_config = getattr(model, "config", None)core_config = getattr(core, "config", None)# A sub-config without its own dtype leaves correct_dtype None, so bnb quant_state.dtype becomes None.wrapper_dtype = getattr(wrapper_config, "dtype", None)- # A child built in __init__ has no _name_or_path: adapter_config and merged saves need the repo.- wrapper_path = getattr(wrapper_config, "_name_or_path", "") or getattr(- model, "name_or_path", ""- )- if core_config is not None and wrapper_path and not getattr(core_config, "_name_or_path", ""):- core_config._name_or_path = wrapper_path- try:- core.name_or_path = wrapper_path- except Exception:- passif (core_config is not Noneand wrapper_dtype is not None@@ -1481,6 +1649,12 @@ def _carry_loader_state_to_core(model, core, name):core_config.dtype = wrapper_dtypeexcept Exception:pass+ # A sub-config names no architecture; generate and save read it off the core.+ if core_config is not None and not getattr(core_config, "architectures", None):+ try:+ core_config.architectures = [type(core).__name__]+ except Exception:+ pass# generation_config.json is loaded onto the wrapper only; the child has config defaults (EOS etc).wrapper_generation_config = getattr(model, "generation_config", None)if wrapper_generation_config is not None:@@ -2686,6 +2860,16 @@ class FastBaseModel:)else:_audio_kwargs = {}+ # No language tag matches `thinker.model`; without this LoRA hits only the vision tower.+ _core_prefix = _text_core_decoder_prefix(model)+ if _core_prefix and "language_tags" in inspect.signature(get_peft_regex).parameters:+ _language_tag_default = (+ inspect.signature(get_peft_regex).parameters["language_tags"].default+ )+ if isinstance(_language_tag_default, (list, tuple)):+ _audio_kwargs["language_tags"] = list(_language_tag_default) + [+ re.escape(_core_prefix)+ ]# Remember the caller's ORIGINAL explicit leaf list for MoE expert detection: routing it through get_peft_regex adds the full "mlp|feed_forward|ffn|dense" block even for attention-only leaves, so keying on that regex would train the experts. Only the auto path relies on the regex._moe_detect_target = target_modules if type(target_modules) in (list, tuple) else None@@ -2713,6 +2897,7 @@ class FastBaseModel:f"`modules_to_save`, so they are trained as full weight matrices.\n"f"This uses more VRAM than LoRA. Please list them in `modules_to_save` directly.")+ modules_to_save = _scope_modules_to_save_to_core(model, modules_to_save)_raise_if_fast_inference_modules_to_save(model, modules_to_save)# Only a regex generated here may be widened to expert submodules below.@@ -2736,7 +2921,14 @@ class FastBaseModel:or not finetune_attention_modulesor not finetune_mlp_modules)- if type(target_modules) in (list, tuple) and (_scoping or finetune_audio_layers):+ # A kept wrapper (Qwen3-Omni) also holds a talker the forward never reaches: scope leaves to the text core.+ _leaf_list = type(target_modules) in (list, tuple) and all(+ "." not in str(name) for name in target_modules+ )+ # Qualified paths already name their module; get_peft_regex treats every entry as a leaf.+ if type(target_modules) in (list, tuple) and (+ _scoping or finetune_audio_layers or (_core_prefix and _leaf_list)+ ):if _scoping:print("Unsloth: Explicit target_modules are constrained by the "@@ -2809,12 +3001,26 @@ class FastBaseModel:_moe_module_targets = get_moe_target_modules(model, _moe_module_detect)# Auto-detect MoE models and populate target_parameters for expert layers.- if target_parameters is None:+ if target_parameters is not None:+ target_parameters = _scope_parameter_targets_to_core(model, target_parameters)+ else:target_parameters = get_moe_target_parameters(model,_moe_module_detect,moe_module_targets = _moe_module_targets,)+ # A forward-less wrapper's config hides the thinker's experts; resolve on the core and+ # name full paths, since PEFT suffix-matches and would also hit the talker's experts.+ _core_name = getattr(model, "_unsloth_text_core", None)+ _core = getattr(model, _core_name, None) if isinstance(_core_name, str) else None+ if target_parameters is None and _core is not None:+ _core_parameters = get_moe_target_parameters(_core, _moe_module_detect)+ if _core_parameters:+ target_parameters = [+ f"{_core_name}.{name}"+ for name, _ in _core.named_parameters()+ if any(name == t or name.endswith("." + t) for t in _core_parameters)+ ] or Noneif _moe_module_targets:if isinstance(target_modules, (list, tuple)):@@ -2837,6 +3043,10 @@ class FastBaseModel:if finetune_last_n_layers is not None and layers_to_transform is None:_total_layers = _get_total_transformer_layers(model)+ _core_name = getattr(model, "_unsloth_text_core", None)+ if _total_layers is None and isinstance(_core_name, str):+ # A kept wrapper's config nests the count (Qwen3-Omni: thinker.config.text_config).+ _total_layers = _get_total_transformer_layers(getattr(model, _core_name, None))if _total_layers is not None and _total_layers > 0:n = max(1, min(int(finetune_last_n_layers), _total_layers))layers_to_transform = list(range(_total_layers - n, _total_layers))@@ -2991,6 +3201,9 @@ class FastBaseModel:float32_mixed_precision = float32_mixed_precision,patch_modules_to_save = True,)+ if full_finetuning:+ # prepare_model_for_training re-enabled every parameter, a kept wrapper's siblings too.+ _freeze_unused_siblings(model)# Persist the configured GC mode so the trainer restores it verbatim: for_inference() clears the module flags every GRPO generation step, and TrainingArguments defaults gradient_checkpointing=False, which would silently disable it at train time (#4735).model._unsloth_gradient_checkpointing = use_gradient_checkpointing