Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fine-tune Whisper, Canary, Moonshine, CTC, audio classification, Dia and Higgs Audio through FastModel by default (#14744)

Daniel Han4b9c85f985c8
diff --git a/tests/python/test_audio_processor_load.py b/tests/python/test_audio_processor_load.py
new file mode 100644
index 00000000000..d5fe0ec49da
--- /dev/null
+++ b/tests/python/test_audio_processor_load.py
@@ -0,0 +1,164 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+
+"""Audio processors load as processors, not as their text tokenizer (Dia, Higgs Audio, Granite Speech)."""
+
+import ast
+import re
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+VISION_PATH = REPO_ROOT / "unsloth" / "models" / "vision.py"
+
+
+def _helpers():
+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))
+ wanted = {"_from_pretrained_left_padded", "_missing_audio_backend"}
+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in wanted]
+ ns = {"re": re}
+ exec(compile(ast.Module(nodes, []), str(VISION_PATH), "exec"), ns)
+ return ns["_from_pretrained_left_padded"], ns["_missing_audio_backend"]
+
+
+class _Loader:
+ """Mimics a processor whose from_pretrained forwards padding_side into a codec model's __init__."""
+
+ def __init__(self, error = None):
+ self.calls = []
+ self.error = error
+
+ def from_pretrained(self, name, **kwargs):
+ self.calls.append(kwargs)
+ if "padding_side" in kwargs and self.error is not None:
+ raise self.error
+ return ("processor", name)
+
+
+def test_padding_side_is_dropped_only_when_it_breaks_the_load():
+ left_padded, _ = _helpers()
+ plain = _Loader()
+ assert left_padded(plain, "repo", token = None) == ("processor", "repo")
+ assert plain.calls == [{"padding_side": "left", "token": None}]
+
+ codec = _Loader(
+ TypeError("DacModel.__init__() got an unexpected keyword argument 'padding_side'")
+ )
+ assert left_padded(codec, "repo", token = None) == ("processor", "repo")
+ assert codec.calls == [{"padding_side": "left", "token": None}, {"token": None}]
+
+ other = _Loader(TypeError("expected str, bytes or os.PathLike object, not NoneType"))
+ with pytest.raises(TypeError, match = "PathLike"):
+ left_padded(other, "repo")
+
+
+def test_missing_audio_backend_names_the_library():
+ _, missing = _helpers()
+ error = ImportError(
+ "\nGraniteSpeechFeatureExtractor requires the torchaudio library but it was not found in your "
+ "environment. Please install it and restart your\nruntime."
+ )
+ assert missing(error) == "torchaudio"
+ assert missing(ImportError("No module named 'foo'")) is None
+ assert missing(ValueError("requires the torchaudio library")) is None
+ assert missing(None) is None
+
+
+def test_processor_loads_use_the_retry():
+ source = VISION_PATH.read_text(encoding = "utf-8")
+ body = source[source.index("def _acquire_processor(lfo):") :]
+ body = body[: body.index("return _tok, _err")]
+ # Every primary auto_processor load goes through the retry; none passes padding_side itself.
+ assert body.count("_from_pretrained_left_padded(") == 2
+ assert "auto_processor.from_pretrained(" not in body
+
+
+def test_real_dia_processor_rejects_padding_side():
+ """The defect being worked around, against the installed transformers (skipped offline)."""
+ transformers = pytest.importorskip("transformers")
+ if not hasattr(transformers, "DiaProcessor"):
+ pytest.skip("transformers has no Dia")
+ left_padded, _ = _helpers()
+ try:
+ processor = left_padded(transformers.AutoProcessor, "nari-labs/Dia-1.6B-0626")
+ except OSError:
+ pytest.skip("Hub not reachable")
+ assert type(processor).__name__ == "DiaProcessor"
+
+
+def _is_speech_forward():
+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))
+ nodes = [
+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_is_speech_forward"
+ ]
+ import inspect
+
+ ns = {"inspect": inspect}
+ exec(compile(ast.Module(nodes, []), str(VISION_PATH), "exec"), ns)
+ return ns["_is_speech_forward"]
+
+
+def _tiny_whisper():
+ torch = pytest.importorskip("torch")
+ transformers = pytest.importorskip("transformers")
+ config = transformers.WhisperConfig(
+ vocab_size = 64,
+ d_model = 32,
+ encoder_layers = 1,
+ decoder_layers = 1,
+ encoder_attention_heads = 2,
+ decoder_attention_heads = 2,
+ encoder_ffn_dim = 64,
+ decoder_ffn_dim = 64,
+ num_mel_bins = 16,
+ max_source_positions = 8,
+ max_target_positions = 16,
+ pad_token_id = 0,
+ bos_token_id = 1,
+ eos_token_id = 2,
+ decoder_start_token_id = 1,
+ )
+ torch.manual_seed(0)
+ model = transformers.WhisperForConditionalGeneration(config)
+ batch = {"input_features": torch.randn(2, 16, 16), "labels": torch.randint(3, 64, (2, 5))}
+ return model, batch
+
+
+def test_speech_models_get_no_task_wrapper():
+ is_speech = _is_speech_forward()
+ transformers = pytest.importorskip("transformers")
+ for cls in (
+ "WhisperForConditionalGeneration",
+ "Wav2Vec2ForCTC",
+ "MoonshineForConditionalGeneration",
+ ):
+ cls = getattr(transformers, cls)
+ assert is_speech(cls.__new__(cls))
+ for cls in ("LlamaForCausalLM", "Qwen2AudioForConditionalGeneration"):
+ cls = getattr(transformers, cls)
+ assert not is_speech(cls.__new__(cls))
+
+ source = VISION_PATH.read_text(encoding = "utf-8")
+ assert "_is_speech_forward(model)" in source
+ assert "task_type = None" in source
+
+
+def test_causal_lm_wrapper_breaks_whisper_training():
+ """The defect: PeftModelForCausalLM passes input_ids = None into Whisper's **kwargs."""
+ peft = pytest.importorskip("peft")
+ model, batch = _tiny_whisper()
+ broken = peft.get_peft_model(
+ model, peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"], task_type = "CAUSAL_LM")
+ )
+ with pytest.raises(TypeError, match = "multiple values for keyword argument 'input_ids'"):
+ broken(**batch)
+
+ model, batch = _tiny_whisper()
+ fixed = peft.get_peft_model(
+ model, peft.LoraConfig(r = 4, target_modules = ["q_proj", "v_proj"], task_type = None)
+ )
+ loss = fixed(**batch).loss
+ loss.backward()
+ assert loss.isfinite()
diff --git a/tests/python/test_feature_extractor_only_checkpoints.py b/tests/python/test_feature_extractor_only_checkpoints.py
new file mode 100644
index 00000000000..c285a2e4e14
--- /dev/null
+++ b/tests/python/test_feature_extractor_only_checkpoints.py
@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+
+"""Encoder-only audio checkpoints (AST, WavLM, wav2vec2 bases) ship only a feature extractor: FastModel
+returns it instead of failing on the missing tokenizer, and keeps it right padded."""
+
+import ast
+from pathlib import Path
+
+import pytest
+
+VISION_PATH = Path(__file__).resolve().parents[2] / "unsloth" / "models" / "vision.py"
+
+
+def _is_bare_feature_extractor():
+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))
+ node = next(
+ n
+ for n in tree.body
+ if isinstance(n, ast.FunctionDef) and n.name == "_is_bare_feature_extractor"
+ )
+ ns = {}
+ exec(compile(ast.Module([node], []), str(VISION_PATH), "exec"), ns)
+ return ns["_is_bare_feature_extractor"]
+
+
+def test_bare_feature_extractor_detection():
+ transformers = pytest.importorskip("transformers")
+ check = _is_bare_feature_extractor()
+ fe = transformers.Wav2Vec2FeatureExtractor()
+ assert check(fe)
+ tok_cls = getattr(transformers, "PreTrainedTokenizerBase")
+ assert not check(object.__new__(tok_cls))
+ assert not check(None)
+
+
+def test_last_resort_tries_feature_extractor_and_keeps_padding():
+ source = VISION_PATH.read_text(encoding = "utf-8")
+ assert "AutoFeatureExtractor.from_pretrained(" in source
+ # AutoProcessor's fallback keeps the padding_side = "left" it was given; waveforms stay right padded.
+ assert (
+ 'tokenizer.padding_side = "right" if _is_bare_feature_extractor(tokenizer) else _padding_side'
+ in source
+ )
+
+
+def _config_maps_audio_auto_class():
+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))
+ wanted = {"_AUDIO_AUTO_CLASS_NAMES", "_config_maps_audio_auto_class"}
+ nodes = [
+ n
+ for n in tree.body
+ if (isinstance(n, ast.FunctionDef) and n.name in wanted)
+ or (isinstance(n, ast.Assign) and any(getattr(t, "id", None) in wanted for t in n.targets))
+ ]
+ ns = {}
+ exec(compile(ast.Module(nodes, []), str(VISION_PATH), "exec"), ns)
+ return ns["_config_maps_audio_auto_class"]
+
+
+def test_audio_configs_get_their_processor():
+ transformers = pytest.importorskip("transformers")
+ maps = _config_maps_audio_auto_class()
+ for cfg in ("Wav2Vec2Config", "HubertConfig", "WhisperConfig", "MoonshineConfig", "ASTConfig"):
+ if hasattr(transformers, cfg):
+ assert maps(getattr(transformers, cfg)()), cfg
+ for cfg in ("LlamaConfig", "Qwen2Config", "T5Config"):
+ assert not maps(getattr(transformers, cfg)()), cfg
+
+
+def test_tokenizer_fallback_keeps_audio_processors():
+ source = VISION_PATH.read_text(encoding = "utf-8")
+ # Granite Speech names its audio frontend audio_processor.
+ assert '("image_processor", "feature_extractor", "audio_processor")' in source
+
+
+LOADER_PATH = VISION_PATH.with_name("loader.py")
+
+
+def _resolve_audio_auto_model():
+ transformers = pytest.importorskip("transformers")
+ tree = ast.parse(LOADER_PATH.read_text(encoding = "utf-8"))
+ wanted = {"_AUDIO_HEAD_AUTO_CLASS_NAMES", "_resolve_audio_auto_model"}
+ nodes = [
+ n
+ for n in tree.body
+ if (isinstance(n, ast.FunctionDef) and n.name in wanted)
+ or (isinstance(n, ast.Assign) and any(getattr(t, "id", None) in wanted for t in n.targets))
+ ]
+
+ def resolve_model_class(auto_model, config, **kwargs):
+ try:
+ return auto_model._model_mapping[type(config)]
+ except KeyError:
+ return None
+
+ ns = {"resolve_model_class": resolve_model_class}
+ exec(compile(ast.Module(nodes, []), str(LOADER_PATH), "exec"), ns)
+ return ns["_resolve_audio_auto_model"], transformers
+
+
+def test_default_auto_model_loads_the_checkpoints_own_audio_head():
+ resolve, transformers = _resolve_audio_auto_model()
+ cases = [
+ (transformers.Wav2Vec2Config(architectures = ["Wav2Vec2ForCTC"]), "AutoModelForCTC"),
+ (
+ transformers.ASTConfig(architectures = ["ASTForAudioClassification"]),
+ "AutoModelForAudioClassification",
+ ),
+ (transformers.WavLMConfig(architectures = ["WavLMModel"]), "AutoModel"),
+ (transformers.WavLMConfig(architectures = ["WavLMForXVector"]), "AutoModelForAudioXVector"),
+ (
+ transformers.WhisperConfig(architectures = ["WhisperForAudioClassification"]),
+ "AutoModelForAudioClassification",
+ ),
+ ]
+ for config, expected in cases:
+ assert resolve(config).__name__ == expected, config.architectures
+ assert resolve(transformers.LlamaConfig(architectures = ["LlamaForCausalLM"])) is None
+ # with a causal-LM mapping, a bare checkpoint keeps today's route (Qwen2Model embedding repos)
+ assert (
+ resolve(transformers.LlamaConfig(architectures = ["LlamaModel"]), include_bare = False) is None
+ )
+ assert resolve(transformers.Wav2Vec2Config()) is None # no architectures: keep today's route
+ source = LOADER_PATH.read_text(encoding = "utf-8")
+ flat = "".join(source.split())
+ assert "_resolve_audio_auto_model(model_config,**_probe_hub_kwargs)" in flat
+ assert "include_bare=resolve_model_class(auto_model,model_config,**_probe_hub_kwargs)" in flat
+
+
+def test_audio_processor_frontends_are_right_padded_too():
+ source = VISION_PATH.read_text(encoding = "utf-8")
+ assert 'for _audio_attr in ("feature_extractor", "audio_processor"):' in source
diff --git a/tests/python/test_peft_non_generative_heads.py b/tests/python/test_peft_non_generative_heads.py
new file mode 100644
index 00000000000..a135026af5b
--- /dev/null
+++ b/tests/python/test_peft_non_generative_heads.py
@@ -0,0 +1,162 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+
+"""get_peft_model on heads that cannot generate (Wav2Vec2ForCTC) uses PEFT's generic PeftModel."""
+
+import ast
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+VISION_PATH = REPO_ROOT / "unsloth" / "models" / "vision.py"
+
+
+def _get_peft_model_source():
+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))
+ for node in tree.body:
+ if isinstance(node, ast.ClassDef) and node.name == "FastBaseModel":
+ for item in node.body:
+ if isinstance(item, ast.FunctionDef) and item.name == "get_peft_model":
+ return ast.unparse(item)
+ raise AssertionError("FastBaseModel.get_peft_model not found")
+
+
+def test_non_generative_heads_drop_the_causal_lm_task_type():
+ source = _get_peft_model_source()
+ assert "prepare_inputs_for_generation" in source
+ assert "task_type = None" in source
+
+
+def _tiny_ctc():
+ torch = pytest.importorskip("torch")
+ transformers = pytest.importorskip("transformers")
+ config = transformers.Wav2Vec2Config(
+ vocab_size = 16,
+ hidden_size = 32,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 64,
+ conv_dim = (16, 16),
+ conv_stride = (5, 2),
+ conv_kernel = (10, 3),
+ num_feat_extract_layers = 2,
+ num_conv_pos_embeddings = 16,
+ num_conv_pos_embedding_groups = 2,
+ )
+ torch.manual_seed(0)
+ return transformers.Wav2Vec2ForCTC(config)
+
+
+def test_ctc_head_trains_as_peft_model_but_not_as_causal_lm():
+ torch = pytest.importorskip("torch")
+ peft = pytest.importorskip("peft")
+ targets = r".*\.attention\.(?:q_proj|v_proj)"
+
+ model = _tiny_ctc()
+ assert not hasattr(model, "prepare_inputs_for_generation")
+ with pytest.raises(AttributeError, match = "prepare_inputs_for_generation"):
+ peft.get_peft_model(
+ model, peft.LoraConfig(r = 4, target_modules = targets, task_type = "CAUSAL_LM")
+ )
+
+ model = peft.get_peft_model(
+ _tiny_ctc(), peft.LoraConfig(r = 4, target_modules = targets, task_type = None)
+ )
+ audio = torch.randn(2, 4000)
+ labels = torch.randint(1, 15, (2, 3))
+ model(input_values = audio, labels = labels).loss.backward()
+ grads = [p.grad for n, p in model.named_parameters() if "lora_B" in n]
+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)
+
+
+def test_causal_lms_keep_the_generation_hook():
+ transformers = pytest.importorskip("transformers")
+ config = transformers.LlamaConfig(
+ vocab_size = 32,
+ hidden_size = 16,
+ intermediate_size = 32,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ )
+ assert hasattr(transformers.LlamaForCausalLM(config), "prepare_inputs_for_generation")
+
+
+def _helper(name):
+ import torch
+
+ tree = ast.parse(VISION_PATH.read_text(encoding = "utf-8"))
+ wanted = {name, "_is_audio_head_model", "_AUDIO_HEAD_SUFFIXES"}
+ nodes = [
+ n
+ for n in tree.body
+ if (isinstance(n, ast.FunctionDef) and n.name in wanted)
+ or (isinstance(n, ast.Assign) and any(getattr(t, "id", None) in wanted for t in n.targets))
+ ]
+ ns = {"torch": torch}
+ exec(compile(ast.Module(nodes, []), str(VISION_PATH), "exec"), ns)
+ return ns[name]
+
+
+def test_ctc_and_classifier_heads_are_trained_and_saved():
+ transformers = pytest.importorskip("transformers")
+ peft = pytest.importorskip("peft")
+ heads = _helper("_trainable_head_modules")
+ assert heads(_tiny_ctc()) == ["lm_head"]
+
+ config = transformers.ASTConfig(
+ hidden_size = 16,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ intermediate_size = 32,
+ num_mel_bins = 16,
+ max_length = 32,
+ num_labels = 3,
+ )
+ classifier = transformers.ASTForAudioClassification(config)
+ assert heads(classifier) == ["classifier"]
+ assert heads(transformers.ASTModel(config)) == [] # a bare encoder is its own backbone
+
+ whisper = transformers.WhisperConfig(
+ vocab_size = 64,
+ d_model = 16,
+ encoder_layers = 1,
+ decoder_layers = 1,
+ encoder_attention_heads = 2,
+ decoder_attention_heads = 2,
+ encoder_ffn_dim = 32,
+ decoder_ffn_dim = 32,
+ num_mel_bins = 8,
+ max_source_positions = 8,
+ num_labels = 3,
+ )
+ # backbone under `encoder`, not base_model_prefix
+ assert heads(transformers.WhisperForAudioClassification(whisper)) == ["projector", "classifier"]
+
+ xvector = _tiny_ctc().config
+ xvector.tdnn_dim, xvector.tdnn_kernel, xvector.tdnn_dilation = (8, 8), (3, 1), (1, 1)
+ xvector.xvector_output_dim = 8
+ names = heads(transformers.Wav2Vec2ForXVector(xvector))
+ assert (
+ "feature_extractor" not in names and "classifier" in names
+ ) # PEFT would also match wav2vec2.feature_extractor
+
+ model = peft.get_peft_model(
+ _tiny_ctc(),
+ peft.LoraConfig(r = 4, target_modules = r".*\.attention\.q_proj", modules_to_save = ["lm_head"]),
+ )
+ trainable = {n for n, p in model.named_parameters() if p.requires_grad}
+ assert any("lm_head" in n for n in trainable)
+ source = _get_peft_model_source()
+ assert "modules_to_save = _trainable_head_modules(model) or None" in source
+ assert (
+ "_cannot_generate or _is_audio_head_model(model)" in source
+ ) # ParakeetForCTC can generate
+ assert "type(model) is PeftModel" in source # a generic PeftModel is not wrapped twice
+
+
+def test_num_labels_on_an_audio_classifier_loads_the_audio_head():
+ source = VISION_PATH.with_name("loader.py").read_text(encoding = "utf-8")
+ shortcut = source[source.index("if _num_labels is not None:") :][:900]
+ assert "AutoModelForAudioClassification" in shortcut
diff --git a/tests/python/test_revision_forwarding.py b/tests/python/test_revision_forwarding.py
index 9b1cb5de7d6..c5d971ed347 100644
--- a/tests/python/test_revision_forwarding.py
+++ b/tests/python/test_revision_forwarding.py
@@ -126,6 +126,12 @@ def test_vision_loads_forward_revision(callee, minimum):
calls = [
c for c in _calls(function, "from_pretrained") if ast.unparse(c.func).startswith(callee)
]
+ # Processor loads go through the padding_side retry, which forwards its kwargs.
+ calls += [
+ c
+ for c in _calls(function, "_from_pretrained_left_padded")
+ if c.args and ast.unparse(c.args[0]) == callee
+ ]
assert len(calls) >= minimum, f"expected >= {minimum} {callee} loads, found {len(calls)}"
for call in calls:
assert _revision_kwarg(call) is not None, f"{callee} at line {call.lineno} drops revision"
diff --git a/tests/test_repo_code_vlm_processor.py b/tests/test_repo_code_vlm_processor.py
index d9f5b7d70c3..76331d35194 100644
--- a/tests/test_repo_code_vlm_processor.py
+++ b/tests/test_repo_code_vlm_processor.py
@@ -94,6 +94,10 @@ def _select(
"AutoModelForImageTextToText": transformers.AutoModelForImageTextToText,
"AutoModelForSpeechSeq2Seq": transformers.AutoModelForSpeechSeq2Seq,
"_multimodal_auto_classes": lambda: (),
+ "_config_maps_audio_auto_class": lambda config: False,
+ "_from_pretrained_left_padded": lambda loader, name, **kw: loader.from_pretrained(
+ name, padding_side = "left", **kw
+ ),
"AutoProcessor": fake_processor,
"AutoTokenizer": fake_tokenizer,
"tokenizer_name": "org/repo",
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 96879177d79..ab3e80e675e 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -774,6 +774,44 @@ def _resolve_speech_seq2seq_auto_model(
return None
+_AUDIO_HEAD_AUTO_CLASS_NAMES = (
+ "AutoModelForCTC",
+ "AutoModelForTDT",
+ "AutoModelForAudioClassification",
+ "AutoModelForAudioFrameClassification",
+ "AutoModelForAudioXVector",
+ "AutoModel",
+)
+
+
+def _resolve_audio_auto_model(
+ model_config,
+ include_bare = True,
+ trust_remote_code = None,
+ **hub_kwargs,
+):
+ """The auto class whose mapped class is the checkpoint's own architecture (Wav2Vec2ForCTC,
+ ASTForAudioClassification, a bare WavLMModel), else None."""
+ import transformers
+
+ architectures = getattr(model_config, "architectures", None) or ()
+ if not architectures:
+ return None
+ for name in _AUDIO_HEAD_AUTO_CLASS_NAMES:
+ if name == "AutoModel" and not include_bare:
+ continue
+ auto_class = getattr(transformers, name, None)
+ try:
+ model_class = auto_class and resolve_model_class(
+ auto_class, model_config, trust_remote_code = trust_remote_code, **hub_kwargs
+ )
+ except Exception:
+ model_class = None
+ if model_class is not None and getattr(model_class, "__name__", None) == architectures[0]:
+ return auto_class
+ return None
+
+
def _get_user_task_config_attrs(user_config):
if user_config is None:
return {}
@@ -3091,6 +3129,14 @@ class FastModel(FastBaseModel):
if _num_labels is not None:
from transformers import AutoModelForSequenceClassification
auto_model = AutoModelForSequenceClassification
+ # Audio classifiers (AST, wav2vec2) map only under the audio classification auto class.
+ if resolve_model_class(auto_model, model_config, **_probe_hub_kwargs) is None:
+ import transformers as _transformers
+ _audio_cls = getattr(_transformers, "AutoModelForAudioClassification", None)
+ if _audio_cls is not None and resolve_model_class(
+ _audio_cls, model_config, **_probe_hub_kwargs
+ ):
+ auto_model = _audio_cls
elif _is_text_seq2seq_config(model_config) or (
# SeamlessM4T: Seq2SeqLM-mapped (text-to-text) beside its speech classes, no causal-LM class.
not is_vlm
@@ -3149,10 +3195,24 @@ class FastModel(FastBaseModel):
auto_model = (
_resolve_omni_auto_model(model_config, **_probe_hub_kwargs)
or _resolve_speech_seq2seq_auto_model(model_config, **_probe_hub_kwargs)
+ or _resolve_audio_auto_model(model_config, **_probe_hub_kwargs)
or auto_model
)
else:
auto_model = AutoModelForCausalLM
+ # CTC / audio classification / bare audio encoders load their own head. A causal-LM mapping
+ # (WhisperConfig -> WhisperForCausalLM) yields only to an audio head the checkpoint names.
+ auto_model = (
+ _resolve_audio_auto_model(
+ model_config,
+ include_bare = resolve_model_class(
+ auto_model, model_config, **_probe_hub_kwargs
+ )
+ is None,
+ **_probe_hub_kwargs,
+ )
+ or auto_model
+ )
load_in_4bit_kwargs = load_in_4bit
load_in_8bit_kwargs = load_in_8bit
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 02d59db2a7b..d23ed5ce368 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -88,6 +88,37 @@ def _is_text_seq2seq_config(config):
return not any(mapped(auto_class) for auto_class in _multimodal_auto_classes())
+_AUDIO_AUTO_CLASS_NAMES = (
+ "AutoModelForSpeechSeq2Seq",
+ "AutoModelForCTC",
+ "AutoModelForTDT",
+ "AutoModelForAudioClassification",
+ "AutoModelForAudioFrameClassification",
+ "AutoModelForAudioXVector",
+ "AutoModelForTextToSpectrogram",
+ "AutoModelForTextToWaveform",
+)
+
+
+def _config_maps_audio_auto_class(config):
+ import transformers
+ for name in _AUDIO_AUTO_CLASS_NAMES:
+ try:
+ if type(config) in getattr(transformers, name)._model_mapping:
+ return True
+ except Exception:
+ continue
+ return False
+
+
+def _is_bare_feature_extractor(processor):
+ try:
+ from transformers.feature_extraction_utils import FeatureExtractionMixin
+ except Exception:
+ return False
+ return isinstance(processor, FeatureExtractionMixin)
+
+
def _generation_padding_side(config):
# BART / Marian encoder positions are absolute, so left padding would shift every real token.
# Any encoder-decoder (Whisper, SeamlessM4T): left-padded labels put pads before the decoder's target.
@@ -204,7 +235,7 @@ except ImportError: # older unsloth_zoo
import torch.utils.checkpoint as torch_checkpoint
import transformers.modeling_utils as hf_modeling_utils
from peft import LoraConfig, TaskType, get_peft_model as _get_peft_model
-from peft import PeftModelForCausalLM, PeftModelForSeq2SeqLM
+from peft import PeftModel, PeftModelForCausalLM, PeftModelForSeq2SeqLM
from transformers import set_seed as transformers_set_seed
from unsloth_zoo.peft_utils import (
get_peft_regex,
@@ -2051,6 +2082,79 @@ def _missing_torchvision_error(error = None):
return False
+_AUDIO_HEAD_SUFFIXES = (
+ "ForCTC",
+ "ForTDT",
+ "ForAudioClassification",
+ "ForSequenceClassification",
+ "ForAudioFrameClassification",
+ "ForXVector",
+)
+
+
+def _is_audio_head_model(model):
+ """A CTC / TDT / audio classification task model, whose head is usually freshly initialized."""
+ return type(model).__name__.endswith(_AUDIO_HEAD_SUFFIXES)
+
+
+def _trainable_head_modules(model):
+ """Top-level heads beside the backbone (CTC lm_head / ctc_head, audio classifier / projector) that hold
+ unquantized weights. PEFT suffix-matches modules_to_save, so a name the backbone also uses is left out."""
+ prefix = getattr(model, "base_model_prefix", None)
+ backbone = getattr(model, prefix, None) if prefix else None
+ children = [(n, c) for n, c in model.named_children() if next(c.parameters(), None) is not None]
+ if backbone is None:
+ if not _is_audio_head_model(
+ model
+ ): # a bare encoder (WavLMModel) is its own backbone: LoRA only
+ return []
+ # WhisperForAudioClassification keeps its backbone (a PreTrainedModel) under `encoder`, not base_model_prefix.
+ from transformers import PreTrainedModel
+
+ backbone = next((c for _, c in children if isinstance(c, PreTrainedModel)), None)
+ if backbone is None:
+ return []
+ backbone_names = {n.rsplit(".", 1)[-1] for n, _ in backbone.named_modules() if n}
+ heads = []
+ for name, child in children:
+ if child is backbone or name in backbone_names:
+ continue
+ if any(type(m).__name__ in ("Linear4bit", "Linear8bitLt") for m in child.modules()):
+ continue
+ heads.append(name)
+ return heads
+
+
+def _is_speech_forward(model):
+ """forward reads audio features and no input_ids (Whisper, Canary, Moonshine, wav2vec2, AST)."""
+ try:
+ params = inspect.signature(type(model).forward).parameters
+ except (TypeError, ValueError):
+ return False
+ return "input_ids" not in params and ("input_features" in params or "input_values" in params)
+
+
+def _from_pretrained_left_padded(loader, name, **kwargs):
+ """`loader.from_pretrained(name, padding_side = "left", ...)`, retried without padding_side when the
+ processor forwards it into a sub-model's __init__ (Dia's DacModel, Higgs Audio's tokenizer model).
+ The inner tokenizer is set to left padding after loading either way."""
+ try:
+ return loader.from_pretrained(name, padding_side = "left", **kwargs)
+ except TypeError as error:
+ if "padding_side" not in str(error):
+ raise
+ return loader.from_pretrained(name, **kwargs)
+
+
+def _missing_audio_backend(error):
+ """The library an audio processor needed but could not import (Granite Speech and Fun-ASR-Nano
+ feature extractors require torchaudio), else None."""
+ if not isinstance(error, ImportError):
+ return None
+ match = re.search(r"requires the (\w+) library", str(error))
+ return match.group(1) if match else None
+
+
# Only where class defaults are the checkpoint's settings (Step-3.7 hardcodes them remotely).
_NATIVE_DEFAULT_IMAGE_PROCESSOR_TYPES = frozenset({"step3p7"})
@@ -3265,6 +3369,12 @@ class FastBaseModel:
needs_processor = is_vlm or auto_model in _multimodal_auto_classes()
# FastModel routes Whisper here without auto_model (#2726); audio features need the processor.
needs_processor = needs_processor or auto_model is AutoModelForSpeechSeq2Seq
+ # Speech / CTC / audio-classification models (Moonshine, Parakeet, wav2vec2, AST) read audio
+ # features too; Seq2SeqLM keeps its tokenizer (SeamlessM4T's text-to-text route).
+ needs_processor = needs_processor or (
+ getattr(auto_model, "__name__", "") != "AutoModelForSeq2SeqLM"
+ and _config_maps_audio_auto_class(auto_config)
+ )
# A repo-code VLM may register only AutoModel / AutoModelForCausalLM (DeepSeek-OCR, Nemotron-VL), so auto_model is not a VLM class though the config is a vision model. Keep is_vlm for processor selection, but treat it as a VLM on the vLLM path so a vision_config model is never silently loaded as text-only.
is_vlm_config = (
is_vlm
@@ -4266,9 +4376,9 @@ class FastBaseModel:
"ForConditionalGeneration"
):
try:
- _tok = auto_processor.from_pretrained(
+ _tok = _from_pretrained_left_padded(
+ auto_processor,
tokenizer_name,
- padding_side = "left",
token = token,
language = whisper_language,
task = whisper_task,
@@ -4299,9 +4409,9 @@ class FastBaseModel:
_tok = None
try:
if _tok is None:
- _tok = auto_processor.from_pretrained(
+ _tok = _from_pretrained_left_padded(
+ auto_processor,
tokenizer_name,
- padding_side = "left",
token = token,
trust_remote_code = trust_remote_code,
cache_dir = kwargs.get("cache_dir"),
@@ -4372,6 +4482,19 @@ class FastBaseModel:
f"Unsloth: Warning - VLM processor fallback returned None for model_type={model_type_arch}",
file = sys.stderr,
)
+ _audio_backend = _missing_audio_backend(_primary_err)
+ if (
+ needs_processor
+ and _audio_backend
+ and not any(hasattr(tokenizer, k) for k in ("feature_extractor", "audio_processor"))
+ ):
+ import sys
+ print(
+ f"Unsloth: Warning - the audio processor for `{tokenizer_name}` needs {_audio_backend}, "
+ f"which is not installed, so only its text tokenizer was loaded. Audio training "
+ f"needs the processor: `pip install {_audio_backend}`, then load again.",
+ file = sys.stderr,
+ )
# Backwards compat: if the processor has no chat_template (old saves without chat_template.jinja) but the inner tokenizer does, copy it to the processor.
if (
hasattr(tokenizer, "tokenizer")
@@ -4428,8 +4551,11 @@ class FastBaseModel:
revision = _tokenizer_revision,
)
model, _fallback_tok = patch_tokenizer(model, _fallback_tok)
- # Re-attach as a processor wrapper if the original was a processor.
- if hasattr(tokenizer, "image_processor"):
+ # Re-attach as a processor wrapper if the original was a processor (audio ones included).
+ if any(
+ hasattr(tokenizer, k)
+ for k in ("image_processor", "feature_extractor", "audio_processor")
+ ):
tokenizer.tokenizer = _fallback_tok
else:
tokenizer = _fallback_tok
@@ -4482,6 +4608,20 @@ class FastBaseModel:
_last_resort_err = _e
if not local_files_only and _is_offline_related_error(_e):
raise
+ if tokenizer is None:
+ # Encoder-only audio checkpoints (AST, WavLM, wav2vec2 bases) ship only a feature extractor.
+ try:
+ from transformers import AutoFeatureExtractor
+ tokenizer = AutoFeatureExtractor.from_pretrained(
+ tokenizer_name,
+ token = token,
+ trust_remote_code = trust_remote_code,
+ cache_dir = kwargs.get("cache_dir"),
+ local_files_only = local_files_only,
+ revision = _tokenizer_revision,
+ )
+ except Exception:
+ tokenizer = None
if tokenizer is None:
del model
raise RuntimeError(
@@ -4511,16 +4651,20 @@ class FastBaseModel:
patch_gradient_accumulation_fix(Trainer)
_padding_side = _generation_padding_side(getattr(model, "config", None))
- tokenizer.padding_side = _padding_side
+ # A bare audio feature extractor pads waveforms, which must stay right padded (AutoProcessor's
+ # fallback to it keeps the padding_side = "left" we passed).
+ tokenizer.padding_side = "right" if _is_bare_feature_extractor(tokenizer) else _padding_side
if hasattr(tokenizer, "tokenizer"):
tokenizer.tokenizer.padding_side = _padding_side
# Audio feature extractors must stay right padded: left (a text setting, forwarded by from_pretrained) shifts Whisper mels and desyncs Gemma 4 audio token counts, crashing on transformers < 5.10.
- feature_extractor = getattr(tokenizer, "feature_extractor", None)
- if (
- feature_extractor is not None
- and getattr(feature_extractor, "padding_side", None) == "left"
- ):
- feature_extractor.padding_side = "right"
+ # Granite Speech names its frontend audio_processor.
+ for _audio_attr in ("feature_extractor", "audio_processor"):
+ feature_extractor = getattr(tokenizer, _audio_attr, None)
+ if (
+ feature_extractor is not None
+ and getattr(feature_extractor, "padding_side", None) == "left"
+ ):
+ feature_extractor.padding_side = "right"
m = model
while hasattr(m, "model"):
m.max_seq_length = max_seq_length
@@ -4655,13 +4799,28 @@ class FastBaseModel:
if r <= 0:
raise TypeError(f"Unsloth: Rank of {str(r)} must be larger than 0.")
- if isinstance(model, (PeftModelForCausalLM, PeftModelForSeq2SeqLM)):
+ if (
+ isinstance(model, (PeftModelForCausalLM, PeftModelForSeq2SeqLM))
+ or type(model) is PeftModel
+ ):
raise RuntimeError("Unsloth: You already added LoRA adapters to your model!")
if task_type == TaskType.CAUSAL_LM and _is_seq2seq_lm_config(
getattr(model, "config", None)
):
# Also multimodal encoder-decoders (T5Gemma2), which load through the VLM path.
task_type = TaskType.SEQ_2_SEQ_LM
+ _cannot_generate = task_type == TaskType.CAUSAL_LM and not hasattr(
+ model, "prepare_inputs_for_generation"
+ )
+ if task_type in (TaskType.CAUSAL_LM, TaskType.SEQ_2_SEQ_LM) and (
+ _is_speech_forward(model) or _cannot_generate
+ ):
+ # Speech / CTC / audio classification models: the task wrappers pass input_ids = None (it reaches
+ # Whisper's decoder twice), and PeftModelForCausalLM reads prepare_inputs_for_generation in __init__.
+ task_type = None
+ if (_cannot_generate or _is_audio_head_model(model)) and modules_to_save is None:
+ # The generic PeftModel neither trains nor saves the (often freshly initialized) head.
+ modules_to_save = _trainable_head_modules(model) or None
if _is_text_seq2seq_config(getattr(model, "config", None)):
# No vision tower: FastLanguageModel's finetune_vision_layers=False must not filter the encoder out.
finetune_vision_layers = True