Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fine-tune Whisper, Canary, Moonshine, CTC, audio classification, Dia and Higgs Audio through FastModel by default (#14744)
Daniel Han4b9c85f985c8
diff --git a/tests/python/test_audio_processor_load.py b/tests/python/test_audio_processor_load.pynew file mode 100644index 00000000000..d5fe0ec49da--- /dev/null+++ b/tests/python/test_audio_processor_load.py@@ -0,0 +1,164 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.++"""Audio processors load as processors, not as their text tokenizer (Dia, Higgs Audio, Granite Speech)."""++import ast+import re+from pathlib import Path++import pytest++REPO_ROOT = Path(__file__).resolve().parents[2]+VISION_PATH = REPO_ROOT / "unsloth" / "models" / "vision.py"+++def _helpers():+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))+ wanted = {"_from_pretrained_left_padded", "_missing_audio_backend"}+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in wanted]+ ns = {"re": re}+ exec(compile(ast.Module(nodes, []), str(VISION_PATH), "exec"), ns)+ return ns["_from_pretrained_left_padded"], ns["_missing_audio_backend"]+++class _Loader:+ """Mimics a processor whose from_pretrained forwards padding_side into a codec model's __init__."""++ def __init__(self, error = None):+ self.calls = []+ self.error = error++ def from_pretrained(self, name, **kwargs):+ self.calls.append(kwargs)+ if "padding_side" in kwargs and self.error is not None:+ raise self.error+ return ("processor", name)+++def test_padding_side_is_dropped_only_when_it_breaks_the_load():+ left_padded, _ = _helpers()+ plain = _Loader()+ assert left_padded(plain, "repo", token = None) == ("processor", "repo")+ assert plain.calls == [{"padding_side": "left", "token": None}]++ codec = _Loader(+ TypeError("DacModel.__init__() got an unexpected keyword argument 'padding_side'")+ )+ assert left_padded(codec, "repo", token = None) == ("processor", "repo")+ assert codec.calls == [{"padding_side": "left", "token": None}, {"token": None}]++ other = _Loader(TypeError("expected str, bytes or os.PathLike object, not NoneType"))+ with pytest.raises(TypeError, match = "PathLike"):+ left_padded(other, "repo")+++def test_missing_audio_backend_names_the_library():+ _, missing = _helpers()+ error = ImportError(+ "\nGraniteSpeechFeatureExtractor requires the torchaudio library but it was not found in your "+ "environment. Please install it and restart your\nruntime."+ )+ assert missing(error) == "torchaudio"+ assert missing(ImportError("No module named 'foo'")) is None+ assert missing(ValueError("requires the torchaudio library")) is None+ assert missing(None) is None+++def test_processor_loads_use_the_retry():+ source = VISION_PATH.read_text(encoding = "utf-8")+ body = source[source.index("def _acquire_processor(lfo):") :]+ body = body[: body.index("return _tok, _err")]+ # Every primary auto_processor load goes through the retry; none passes padding_side itself.+ assert body.count("_from_pretrained_left_padded(") == 2+ assert "auto_processor.from_pretrained(" not in body+++def test_real_dia_processor_rejects_padding_side():+ """The defect being worked around, against the installed transformers (skipped offline)."""+ transformers = pytest.importorskip("transformers")+ if not hasattr(transformers, "DiaProcessor"):+ pytest.skip("transformers has no Dia")+ left_padded, _ = _helpers()+ try:+ processor = left_padded(transformers.AutoProcessor, "nari-labs/Dia-1.6B-0626")+ except OSError:+ pytest.skip("Hub not reachable")+ assert type(processor).__name__ == "DiaProcessor"+++def _is_speech_forward():+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))+ nodes = [+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_is_speech_forward"+ ]+ import inspect++ ns = {"inspect": inspect}+ exec(compile(ast.Module(nodes, []), str(VISION_PATH), "exec"), ns)+ return ns["_is_speech_forward"]+++def _tiny_whisper():+ torch = pytest.importorskip("torch")+ transformers = pytest.importorskip("transformers")+ config = transformers.WhisperConfig(+ vocab_size = 64,+ d_model = 32,+ encoder_layers = 1,+ decoder_layers = 1,+ encoder_attention_heads = 2,+ decoder_attention_heads = 2,+ encoder_ffn_dim = 64,+ decoder_ffn_dim = 64,+ num_mel_bins = 16,+ max_source_positions = 8,+ max_target_positions = 16,+ pad_token_id = 0,+ bos_token_id = 1,+ eos_token_id = 2,+ decoder_start_token_id = 1,+ )+ torch.manual_seed(0)+ model = transformers.WhisperForConditionalGeneration(config)+ batch = {"input_features": torch.randn(2, 16, 16), "labels": torch.randint(3, 64, (2, 5))}+ return model, batch+++def test_speech_models_get_no_task_wrapper():+ is_speech = _is_speech_forward()+ transformers = pytest.importorskip("transformers")+ for cls in (+ "WhisperForConditionalGeneration",+ "Wav2Vec2ForCTC",+ "MoonshineForConditionalGeneration",+ ):+ cls = getattr(transformers, cls)+ assert is_speech(cls.__new__(cls))+ for cls in ("LlamaForCausalLM", "Qwen2AudioForConditionalGeneration"):+ cls = getattr(transformers, cls)+ assert not is_speech(cls.__new__(cls))++ source = VISION_PATH.read_text(encoding = "utf-8")+ assert "_is_speech_forward(model)" in source+ assert "task_type = None" in source+++def test_causal_lm_wrapper_breaks_whisper_training():+ """The defect: PeftModelForCausalLM passes input_ids = None into Whisper's **kwargs."""+ peft = pytest.importorskip("peft")+ model, batch = _tiny_whisper()+ broken = peft.get_peft_model(+ model, peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"], task_type = "CAUSAL_LM")+ )+ with pytest.raises(TypeError, match = "multiple values for keyword argument 'input_ids'"):+ broken(**batch)++ model, batch = _tiny_whisper()+ fixed = peft.get_peft_model(+ model, peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"], task_type = None)+ )+ loss = fixed(**batch).loss+ loss.backward()+ assert loss.isfinite()diff --git a/tests/python/test_feature_extractor_only_checkpoints.py b/tests/python/test_feature_extractor_only_checkpoints.pynew file mode 100644index 00000000000..c285a2e4e14--- /dev/null+++ b/tests/python/test_feature_extractor_only_checkpoints.py@@ -0,0 +1,133 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.++"""Encoder-only audio checkpoints (AST, WavLM, wav2vec2 bases) ship only a feature extractor: FastModel+returns it instead of failing on the missing tokenizer, and keeps it right padded."""++import ast+from pathlib import Path++import pytest++VISION_PATH = Path(__file__).resolve().parents[2] / "unsloth" / "models" / "vision.py"+++def _is_bare_feature_extractor():+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))+ node = next(+ n+ for n in tree.body+ if isinstance(n, ast.FunctionDef) and n.name == "_is_bare_feature_extractor"+ )+ ns = {}+ exec(compile(ast.Module([node], []), str(VISION_PATH), "exec"), ns)+ return ns["_is_bare_feature_extractor"]+++def test_bare_feature_extractor_detection():+ transformers = pytest.importorskip("transformers")+ check = _is_bare_feature_extractor()+ fe = transformers.Wav2Vec2FeatureExtractor()+ assert check(fe)+ tok_cls = getattr(transformers, "PreTrainedTokenizerBase")+ assert not check(object.__new__(tok_cls))+ assert not check(None)+++def test_last_resort_tries_feature_extractor_and_keeps_padding():+ source = VISION_PATH.read_text(encoding = "utf-8")+ assert "AutoFeatureExtractor.from_pretrained(" in source+ # AutoProcessor's fallback keeps the padding_side = "left" it was given; waveforms stay right padded.+ assert (+ 'tokenizer.padding_side = "right" if _is_bare_feature_extractor(tokenizer) else _padding_side'+ in source+ )+++def _config_maps_audio_auto_class():+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))+ wanted = {"_AUDIO_AUTO_CLASS_NAMES", "_config_maps_audio_auto_class"}+ nodes = [+ n+ for n in tree.body+ if (isinstance(n, ast.FunctionDef) and n.name in wanted)+ or (isinstance(n, ast.Assign) and any(getattr(t, "id", None) in wanted for t in n.targets))+ ]+ ns = {}+ exec(compile(ast.Module(nodes, []), str(VISION_PATH), "exec"), ns)+ return ns["_config_maps_audio_auto_class"]+++def test_audio_configs_get_their_processor():+ transformers = pytest.importorskip("transformers")+ maps = _config_maps_audio_auto_class()+ for cfg in ("Wav2Vec2Config", "HubertConfig", "WhisperConfig", "MoonshineConfig", "ASTConfig"):+ if hasattr(transformers, cfg):+ assert maps(getattr(transformers, cfg)()), cfg+ for cfg in ("LlamaConfig", "Qwen2Config", "T5Config"):+ assert not maps(getattr(transformers, cfg)()), cfg+++def test_tokenizer_fallback_keeps_audio_processors():+ source = VISION_PATH.read_text(encoding = "utf-8")+ # Granite Speech names its audio frontend audio_processor.+ assert '("image_processor", "feature_extractor", "audio_processor")' in source+++LOADER_PATH = VISION_PATH.with_name("loader.py")+++def _resolve_audio_auto_model():+ transformers = pytest.importorskip("transformers")+ tree = ast.parse(LOADER_PATH.read_text(encoding = "utf-8"))+ wanted = {"_AUDIO_HEAD_AUTO_CLASS_NAMES", "_resolve_audio_auto_model"}+ nodes = [+ n+ for n in tree.body+ if (isinstance(n, ast.FunctionDef) and n.name in wanted)+ or (isinstance(n, ast.Assign) and any(getattr(t, "id", None) in wanted for t in n.targets))+ ]++ def resolve_model_class(auto_model, config, **kwargs):+ try:+ return auto_model._model_mapping[type(config)]+ except KeyError:+ return None++ ns = {"resolve_model_class": resolve_model_class}+ exec(compile(ast.Module(nodes, []), str(LOADER_PATH), "exec"), ns)+ return ns["_resolve_audio_auto_model"], transformers+++def test_default_auto_model_loads_the_checkpoints_own_audio_head():+ resolve, transformers = _resolve_audio_auto_model()+ cases = [+ (transformers.Wav2Vec2Config(architectures = ["Wav2Vec2ForCTC"]), "AutoModelForCTC"),+ (+ transformers.ASTConfig(architectures = ["ASTForAudioClassification"]),+ "AutoModelForAudioClassification",+ ),+ (transformers.WavLMConfig(architectures = ["WavLMModel"]), "AutoModel"),+ (transformers.WavLMConfig(architectures = ["WavLMForXVector"]), "AutoModelForAudioXVector"),+ (+ transformers.WhisperConfig(architectures = ["WhisperForAudioClassification"]),+ "AutoModelForAudioClassification",+ ),+ ]+ for config, expected in cases:+ assert resolve(config).__name__ == expected, config.architectures+ assert resolve(transformers.LlamaConfig(architectures = ["LlamaForCausalLM"])) is None+ # with a causal-LM mapping, a bare checkpoint keeps today's route (Qwen2Model embedding repos)+ assert (+ resolve(transformers.LlamaConfig(architectures = ["LlamaModel"]), include_bare = False) is None+ )+ assert resolve(transformers.Wav2Vec2Config()) is None # no architectures: keep today's route+ source = LOADER_PATH.read_text(encoding = "utf-8")+ flat = "".join(source.split())+ assert "_resolve_audio_auto_model(model_config,**_probe_hub_kwargs)" in flat+ assert "include_bare=resolve_model_class(auto_model,model_config,**_probe_hub_kwargs)" in flat+++def test_audio_processor_frontends_are_right_padded_too():+ source = VISION_PATH.read_text(encoding = "utf-8")+ assert 'for _audio_attr in ("feature_extractor", "audio_processor"):' in sourcediff --git a/tests/python/test_peft_non_generative_heads.py b/tests/python/test_peft_non_generative_heads.pynew file mode 100644index 00000000000..a135026af5b--- /dev/null+++ b/tests/python/test_peft_non_generative_heads.py@@ -0,0 +1,162 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.++"""get_peft_model on heads that cannot generate (Wav2Vec2ForCTC) uses PEFT's generic PeftModel."""++import ast+from pathlib import Path++import pytest++REPO_ROOT = Path(__file__).resolve().parents[2]+VISION_PATH = REPO_ROOT / "unsloth" / "models" / "vision.py"+++def _get_peft_model_source():+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))+ for node in tree.body:+ if isinstance(node, ast.ClassDef) and node.name == "FastBaseModel":+ for item in node.body:+ if isinstance(item, ast.FunctionDef) and item.name == "get_peft_model":+ return ast.unparse(item)+ raise AssertionError("FastBaseModel.get_peft_model not found")+++def test_non_generative_heads_drop_the_causal_lm_task_type():+ source = _get_peft_model_source()+ assert "prepare_inputs_for_generation" in source+ assert "task_type = None" in source+++def _tiny_ctc():+ torch = pytest.importorskip("torch")+ transformers = pytest.importorskip("transformers")+ config = transformers.Wav2Vec2Config(+ vocab_size = 16,+ hidden_size = 32,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ intermediate_size = 64,+ conv_dim = (16, 16),+ conv_stride = (5, 2),+ conv_kernel = (10, 3),+ num_feat_extract_layers = 2,+ num_conv_pos_embeddings = 16,+ num_conv_pos_embedding_groups = 2,+ )+ torch.manual_seed(0)+ return transformers.Wav2Vec2ForCTC(config)+++def test_ctc_head_trains_as_peft_model_but_not_as_causal_lm():+ torch = pytest.importorskip("torch")+ peft = pytest.importorskip("peft")+ targets = r".*\.attention\.(?:q_proj|v_proj)"++ model = _tiny_ctc()+ assert not hasattr(model, "prepare_inputs_for_generation")+ with pytest.raises(AttributeError, match = "prepare_inputs_for_generation"):+ peft.get_peft_model(+ model, peft.LoraConfig(r = 4, target_modules = targets, task_type = "CAUSAL_LM")+ )++ model = peft.get_peft_model(+ _tiny_ctc(), peft.LoraConfig(r = 4, target_modules = targets, task_type = None)+ )+ audio = torch.randn(2, 4000)+ labels = torch.randint(1, 15, (2, 3))+ model(input_values = audio, labels = labels).loss.backward()+ grads = [p.grad for n, p in model.named_parameters() if "lora_B" in n]+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)+++def test_causal_lms_keep_the_generation_hook():+ transformers = pytest.importorskip("transformers")+ config = transformers.LlamaConfig(+ vocab_size = 32,+ hidden_size = 16,+ intermediate_size = 32,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ )+ assert hasattr(transformers.LlamaForCausalLM(config), "prepare_inputs_for_generation")+++def _helper(name):+ import torch++ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))+ wanted = {name, "_is_audio_head_model", "_AUDIO_HEAD_SUFFIXES"}+ nodes = [+ n+ for n in tree.body+ if (isinstance(n, ast.FunctionDef) and n.name in wanted)+ or (isinstance(n, ast.Assign) and any(getattr(t, "id", None) in wanted for t in n.targets))+ ]+ ns = {"torch": torch}+ exec(compile(ast.Module(nodes, []), str(VISION_PATH), "exec"), ns)+ return ns[name]+++def test_ctc_and_classifier_heads_are_trained_and_saved():+ transformers = pytest.importorskip("transformers")+ peft = pytest.importorskip("peft")+ heads = _helper("_trainable_head_modules")+ assert heads(_tiny_ctc()) == ["lm_head"]++ config = transformers.ASTConfig(+ hidden_size = 16,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ intermediate_size = 32,+ num_mel_bins = 16,+ max_length = 32,+ num_labels = 3,+ )+ classifier = transformers.ASTForAudioClassification(config)+ assert heads(classifier) == ["classifier"]+ assert heads(transformers.ASTModel(config)) == [] # a bare encoder is its own backbone++ whisper = transformers.WhisperConfig(+ vocab_size = 64,+ d_model = 16,+ encoder_layers = 1,+ decoder_layers = 1,+ encoder_attention_heads = 2,+ decoder_attention_heads = 2,+ encoder_ffn_dim = 32,+ decoder_ffn_dim = 32,+ num_mel_bins = 8,+ max_source_positions = 8,+ num_labels = 3,+ )+ # backbone under `encoder`, not base_model_prefix+ assert heads(transformers.WhisperForAudioClassification(whisper)) == ["projector", "classifier"]++ xvector = _tiny_ctc().config+ xvector.tdnn_dim, xvector.tdnn_kernel, xvector.tdnn_dilation = (8, 8), (3, 1), (1, 1)+ xvector.xvector_output_dim = 8+ names = heads(transformers.Wav2Vec2ForXVector(xvector))+ assert (+ "feature_extractor" not in names and "classifier" in names+ ) # PEFT would also match wav2vec2.feature_extractor++ model = peft.get_peft_model(+ _tiny_ctc(),+ peft.LoraConfig(r = 4, target_modules = r".*\.attention\.q_proj", modules_to_save = ["lm_head"]),+ )+ trainable = {n for n, p in model.named_parameters() if p.requires_grad}+ assert any("lm_head" in n for n in trainable)+ source = _get_peft_model_source()+ assert "modules_to_save = _trainable_head_modules(model) or None" in source+ assert (+ "_cannot_generate or _is_audio_head_model(model)" in source+ ) # ParakeetForCTC can generate+ assert "type(model) is PeftModel" in source # a generic PeftModel is not wrapped twice+++def test_num_labels_on_an_audio_classifier_loads_the_audio_head():+ source = VISION_PATH.with_name("loader.py").read_text(encoding = "utf-8")+ shortcut = source[source.index("if _num_labels is not None:") :][:900]+ assert "AutoModelForAudioClassification" in shortcutdiff --git a/tests/python/test_revision_forwarding.py b/tests/python/test_revision_forwarding.pyindex 9b1cb5de7d6..c5d971ed347 100644--- a/tests/python/test_revision_forwarding.py+++ b/tests/python/test_revision_forwarding.py@@ -126,6 +126,12 @@ def test_vision_loads_forward_revision(callee, minimum):calls = [c for c in _calls(function, "from_pretrained") if ast.unparse(c.func).startswith(callee)]+ # Processor loads go through the padding_side retry, which forwards its kwargs.+ calls += [+ c+ for c in _calls(function, "_from_pretrained_left_padded")+ if c.args and ast.unparse(c.args[0]) == callee+ ]assert len(calls) >= minimum, f"expected >= {minimum} {callee} loads, found {len(calls)}"for call in calls:assert _revision_kwarg(call) is not None, f"{callee} at line {call.lineno} drops revision"diff --git a/tests/test_repo_code_vlm_processor.py b/tests/test_repo_code_vlm_processor.pyindex d9f5b7d70c3..76331d35194 100644--- a/tests/test_repo_code_vlm_processor.py+++ b/tests/test_repo_code_vlm_processor.py@@ -94,6 +94,10 @@ def _select("AutoModelForImageTextToText": transformers.AutoModelForImageTextToText,"AutoModelForSpeechSeq2Seq": transformers.AutoModelForSpeechSeq2Seq,"_multimodal_auto_classes": lambda: (),+ "_config_maps_audio_auto_class": lambda config: False,+ "_from_pretrained_left_padded": lambda loader, name, **kw: loader.from_pretrained(+ name, padding_side = "left", **kw+ ),"AutoProcessor": fake_processor,"AutoTokenizer": fake_tokenizer,"tokenizer_name": "org/repo",diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 96879177d79..ab3e80e675e 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -774,6 +774,44 @@ def _resolve_speech_seq2seq_auto_model(return None+_AUDIO_HEAD_AUTO_CLASS_NAMES = (+ "AutoModelForCTC",+ "AutoModelForTDT",+ "AutoModelForAudioClassification",+ "AutoModelForAudioFrameClassification",+ "AutoModelForAudioXVector",+ "AutoModel",+)+++def _resolve_audio_auto_model(+ model_config,+ include_bare = True,+ trust_remote_code = None,+ **hub_kwargs,+):+ """The auto class whose mapped class is the checkpoint's own architecture (Wav2Vec2ForCTC,+ ASTForAudioClassification, a bare WavLMModel), else None."""+ import transformers++ architectures = getattr(model_config, "architectures", None) or ()+ if not architectures:+ return None+ for name in _AUDIO_HEAD_AUTO_CLASS_NAMES:+ if name == "AutoModel" and not include_bare:+ continue+ auto_class = getattr(transformers, name, None)+ try:+ model_class = auto_class and resolve_model_class(+ auto_class, model_config, trust_remote_code = trust_remote_code, **hub_kwargs+ )+ except Exception:+ model_class = None+ if model_class is not None and getattr(model_class, "__name__", None) == architectures[0]:+ return auto_class+ return None++def _get_user_task_config_attrs(user_config):if user_config is None:return {}@@ -3091,6 +3129,14 @@ class FastModel(FastBaseModel):if _num_labels is not None:from transformers import AutoModelForSequenceClassificationauto_model = AutoModelForSequenceClassification+ # Audio classifiers (AST, wav2vec2) map only under the audio classification auto class.+ if resolve_model_class(auto_model, model_config, **_probe_hub_kwargs) is None:+ import transformers as _transformers+ _audio_cls = getattr(_transformers, "AutoModelForAudioClassification", None)+ if _audio_cls is not None and resolve_model_class(+ _audio_cls, model_config, **_probe_hub_kwargs+ ):+ auto_model = _audio_clselif _is_text_seq2seq_config(model_config) or (# SeamlessM4T: Seq2SeqLM-mapped (text-to-text) beside its speech classes, no causal-LM class.not is_vlm@@ -3149,10 +3195,24 @@ class FastModel(FastBaseModel):auto_model = (_resolve_omni_auto_model(model_config, **_probe_hub_kwargs)or _resolve_speech_seq2seq_auto_model(model_config, **_probe_hub_kwargs)+ or _resolve_audio_auto_model(model_config, **_probe_hub_kwargs)or auto_model)else:auto_model = AutoModelForCausalLM+ # CTC / audio classification / bare audio encoders load their own head. A causal-LM mapping+ # (WhisperConfig -> WhisperForCausalLM) yields only to an audio head the checkpoint names.+ auto_model = (+ _resolve_audio_auto_model(+ model_config,+ include_bare = resolve_model_class(+ auto_model, model_config, **_probe_hub_kwargs+ )+ is None,+ **_probe_hub_kwargs,+ )+ or auto_model+ )load_in_4bit_kwargs = load_in_4bitload_in_8bit_kwargs = load_in_8bitdiff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 02d59db2a7b..d23ed5ce368 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -88,6 +88,37 @@ def _is_text_seq2seq_config(config):return not any(mapped(auto_class) for auto_class in _multimodal_auto_classes())+_AUDIO_AUTO_CLASS_NAMES = (+ "AutoModelForSpeechSeq2Seq",+ "AutoModelForCTC",+ "AutoModelForTDT",+ "AutoModelForAudioClassification",+ "AutoModelForAudioFrameClassification",+ "AutoModelForAudioXVector",+ "AutoModelForTextToSpectrogram",+ "AutoModelForTextToWaveform",+)+++def _config_maps_audio_auto_class(config):+ import transformers+ for name in _AUDIO_AUTO_CLASS_NAMES:+ try:+ if type(config) in getattr(transformers, name)._model_mapping:+ return True+ except Exception:+ continue+ return False+++def _is_bare_feature_extractor(processor):+ try:+ from transformers.feature_extraction_utils import FeatureExtractionMixin+ except Exception:+ return False+ return isinstance(processor, FeatureExtractionMixin)++def _generation_padding_side(config):# BART / Marian encoder positions are absolute, so left padding would shift every real token.# Any encoder-decoder (Whisper, SeamlessM4T): left-padded labels put pads before the decoder's target.@@ -204,7 +235,7 @@ except ImportError: # older unsloth_zooimport torch.utils.checkpoint as torch_checkpointimport transformers.modeling_utils as hf_modeling_utilsfrom peft import LoraConfig, TaskType, get_peft_model as _get_peft_model-from peft import PeftModelForCausalLM, PeftModelForSeq2SeqLM+from peft import PeftModel, PeftModelForCausalLM, PeftModelForSeq2SeqLMfrom transformers import set_seed as transformers_set_seedfrom unsloth_zoo.peft_utils import (get_peft_regex,@@ -2051,6 +2082,79 @@ def _missing_torchvision_error(error = None):return False+_AUDIO_HEAD_SUFFIXES = (+ "ForCTC",+ "ForTDT",+ "ForAudioClassification",+ "ForSequenceClassification",+ "ForAudioFrameClassification",+ "ForXVector",+)+++def _is_audio_head_model(model):+ """A CTC / TDT / audio classification task model, whose head is usually freshly initialized."""+ return type(model).__name__.endswith(_AUDIO_HEAD_SUFFIXES)+++def _trainable_head_modules(model):+ """Top-level heads beside the backbone (CTC lm_head / ctc_head, audio classifier / projector) that hold+ unquantized weights. PEFT suffix-matches modules_to_save, so a name the backbone also uses is left out."""+ prefix = getattr(model, "base_model_prefix", None)+ backbone = getattr(model, prefix, None) if prefix else None+ children = [(n, c) for n, c in model.named_children() if next(c.parameters(), None) is not None]+ if backbone is None:+ if not _is_audio_head_model(+ model+ ): # a bare encoder (WavLMModel) is its own backbone: LoRA only+ return []+ # WhisperForAudioClassification keeps its backbone (a PreTrainedModel) under `encoder`, not base_model_prefix.+ from transformers import PreTrainedModel++ backbone = next((c for _, c in children if isinstance(c, PreTrainedModel)), None)+ if backbone is None:+ return []+ backbone_names = {n.rsplit(".", 1)[-1] for n, _ in backbone.named_modules() if n}+ heads = []+ for name, child in children:+ if child is backbone or name in backbone_names:+ continue+ if any(type(m).__name__ in ("Linear4bit", "Linear8bitLt") for m in child.modules()):+ continue+ heads.append(name)+ return heads+++def _is_speech_forward(model):+ """forward reads audio features and no input_ids (Whisper, Canary, Moonshine, wav2vec2, AST)."""+ try:+ params = inspect.signature(type(model).forward).parameters+ except (TypeError, ValueError):+ return False+ return "input_ids" not in params and ("input_features" in params or "input_values" in params)+++def _from_pretrained_left_padded(loader, name, **kwargs):+ """`loader.from_pretrained(name, padding_side = "left", ...)`, retried without padding_side when the+ processor forwards it into a sub-model's __init__ (Dia's DacModel, Higgs Audio's tokenizer model).+ The inner tokenizer is set to left padding after loading either way."""+ try:+ return loader.from_pretrained(name, padding_side = "left", **kwargs)+ except TypeError as error:+ if "padding_side" not in str(error):+ raise+ return loader.from_pretrained(name, **kwargs)+++def _missing_audio_backend(error):+ """The library an audio processor needed but could not import (Granite Speech and Fun-ASR-Nano+ feature extractors require torchaudio), else None."""+ if not isinstance(error, ImportError):+ return None+ match = re.search(r"requires the (\w+) library", str(error))+ return match.group(1) if match else None++# Only where class defaults are the checkpoint's settings (Step-3.7 hardcodes them remotely)._NATIVE_DEFAULT_IMAGE_PROCESSOR_TYPES = frozenset({"step3p7"})@@ -3265,6 +3369,12 @@ class FastBaseModel:needs_processor = is_vlm or auto_model in _multimodal_auto_classes()# FastModel routes Whisper here without auto_model (#2726); audio features need the processor.needs_processor = needs_processor or auto_model is AutoModelForSpeechSeq2Seq+ # Speech / CTC / audio-classification models (Moonshine, Parakeet, wav2vec2, AST) read audio+ # features too; Seq2SeqLM keeps its tokenizer (SeamlessM4T's text-to-text route).+ needs_processor = needs_processor or (+ getattr(auto_model, "__name__", "") != "AutoModelForSeq2SeqLM"+ and _config_maps_audio_auto_class(auto_config)+ )# A repo-code VLM may register only AutoModel / AutoModelForCausalLM (DeepSeek-OCR, Nemotron-VL), so auto_model is not a VLM class though the config is a vision model. Keep is_vlm for processor selection, but treat it as a VLM on the vLLM path so a vision_config model is never silently loaded as text-only.is_vlm_config = (is_vlm@@ -4266,9 +4376,9 @@ class FastBaseModel:"ForConditionalGeneration"):try:- _tok = auto_processor.from_pretrained(+ _tok = _from_pretrained_left_padded(+ auto_processor,tokenizer_name,- padding_side = "left",token = token,language = whisper_language,task = whisper_task,@@ -4299,9 +4409,9 @@ class FastBaseModel:_tok = Nonetry:if _tok is None:- _tok = auto_processor.from_pretrained(+ _tok = _from_pretrained_left_padded(+ auto_processor,tokenizer_name,- padding_side = "left",token = token,trust_remote_code = trust_remote_code,cache_dir = kwargs.get("cache_dir"),@@ -4372,6 +4482,19 @@ class FastBaseModel:f"Unsloth: Warning - VLM processor fallback returned None for model_type={model_type_arch}",file = sys.stderr,)+ _audio_backend = _missing_audio_backend(_primary_err)+ if (+ needs_processor+ and _audio_backend+ and not any(hasattr(tokenizer, k) for k in ("feature_extractor", "audio_processor"))+ ):+ import sys+ print(+ f"Unsloth: Warning - the audio processor for `{tokenizer_name}` needs {_audio_backend}, "+ f"which is not installed, so only its text tokenizer was loaded. Audio training "+ f"needs the processor: `pip install {_audio_backend}`, then load again.",+ file = sys.stderr,+ )# Backwards compat: if the processor has no chat_template (old saves without chat_template.jinja) but the inner tokenizer does, copy it to the processor.if (hasattr(tokenizer, "tokenizer")@@ -4428,8 +4551,11 @@ class FastBaseModel:revision = _tokenizer_revision,)model, _fallback_tok = patch_tokenizer(model, _fallback_tok)- # Re-attach as a processor wrapper if the original was a processor.- if hasattr(tokenizer, "image_processor"):+ # Re-attach as a processor wrapper if the original was a processor (audio ones included).+ if any(+ hasattr(tokenizer, k)+ for k in ("image_processor", "feature_extractor", "audio_processor")+ ):tokenizer.tokenizer = _fallback_tokelse:tokenizer = _fallback_tok@@ -4482,6 +4608,20 @@ class FastBaseModel:_last_resort_err = _eif not local_files_only and _is_offline_related_error(_e):raise+ if tokenizer is None:+ # Encoder-only audio checkpoints (AST, WavLM, wav2vec2 bases) ship only a feature extractor.+ try:+ from transformers import AutoFeatureExtractor+ tokenizer = AutoFeatureExtractor.from_pretrained(+ tokenizer_name,+ token = token,+ trust_remote_code = trust_remote_code,+ cache_dir = kwargs.get("cache_dir"),+ local_files_only = local_files_only,+ revision = _tokenizer_revision,+ )+ except Exception:+ tokenizer = Noneif tokenizer is None:del modelraise RuntimeError(@@ -4511,16 +4651,20 @@ class FastBaseModel:patch_gradient_accumulation_fix(Trainer)_padding_side = _generation_padding_side(getattr(model, "config", None))- tokenizer.padding_side = _padding_side+ # A bare audio feature extractor pads waveforms, which must stay right padded (AutoProcessor's+ # fallback to it keeps the padding_side = "left" we passed).+ tokenizer.padding_side = "right" if _is_bare_feature_extractor(tokenizer) else _padding_sideif hasattr(tokenizer, "tokenizer"):tokenizer.tokenizer.padding_side = _padding_side# Audio feature extractors must stay right padded: left (a text setting, forwarded by from_pretrained) shifts Whisper mels and desyncs Gemma 4 audio token counts, crashing on transformers < 5.10.- feature_extractor = getattr(tokenizer, "feature_extractor", None)- if (- feature_extractor is not None- and getattr(feature_extractor, "padding_side", None) == "left"- ):- feature_extractor.padding_side = "right"+ # Granite Speech names its frontend audio_processor.+ for _audio_attr in ("feature_extractor", "audio_processor"):+ feature_extractor = getattr(tokenizer, _audio_attr, None)+ if (+ feature_extractor is not None+ and getattr(feature_extractor, "padding_side", None) == "left"+ ):+ feature_extractor.padding_side = "right"m = modelwhile hasattr(m, "model"):m.max_seq_length = max_seq_length@@ -4655,13 +4799,28 @@ class FastBaseModel:if r <= 0:raise TypeError(f"Unsloth: Rank of {str(r)} must be larger than 0.")- if isinstance(model, (PeftModelForCausalLM, PeftModelForSeq2SeqLM)):+ if (+ isinstance(model, (PeftModelForCausalLM, PeftModelForSeq2SeqLM))+ or type(model) is PeftModel+ ):raise RuntimeError("Unsloth: You already added LoRA adapters to your model!")if task_type == TaskType.CAUSAL_LM and _is_seq2seq_lm_config(getattr(model, "config", None)):# Also multimodal encoder-decoders (T5Gemma2), which load through the VLM path.task_type = TaskType.SEQ_2_SEQ_LM+ _cannot_generate = task_type == TaskType.CAUSAL_LM and not hasattr(+ model, "prepare_inputs_for_generation"+ )+ if task_type in (TaskType.CAUSAL_LM, TaskType.SEQ_2_SEQ_LM) and (+ _is_speech_forward(model) or _cannot_generate+ ):+ # Speech / CTC / audio classification models: the task wrappers pass input_ids = None (it reaches+ # Whisper's decoder twice), and PeftModelForCausalLM reads prepare_inputs_for_generation in __init__.+ task_type = None+ if (_cannot_generate or _is_audio_head_model(model)) and modules_to_save is None:+ # The generic PeftModel neither trains nor saves the (often freshly initialized) head.+ modules_to_save = _trainable_head_modules(model) or Noneif _is_text_seq2seq_config(getattr(model, "config", None)):# No vision tower: FastLanguageModel's finetune_vision_layers=False must not filter the encoder out.finetune_vision_layers = True