Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Support for Seq2Seq Models (T5, T5Gemma, etc.) (#4226)

Daniel Han6d6fbc4bc6ae
diff --git a/tests/test_seq2seq_auto_class.py b/tests/test_seq2seq_auto_class.py
new file mode 100644
index 00000000000..7187cfe19dd
--- /dev/null
+++ b/tests/test_seq2seq_auto_class.py
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Text encoder-decoders load via AutoModelForSeq2SeqLM; multimodal models also mapped there keep their class."""
+
+import pytest
+
+transformers = pytest.importorskip("transformers")
+
+from unsloth.models.vision import _is_text_seq2seq_config # noqa: E402
+
+
+def _config(name):
+ cls = getattr(transformers, name, None)
+ if cls is None:
+ pytest.skip(f"this transformers has no {name}")
+ return cls()
+
+
+ "name",
+ ["T5Config", "MT5Config", "BartConfig", "MarianConfig", "PegasusConfig", "T5GemmaConfig"],
+)
+def test_text_seq2seq_configs_route_to_seq2seq(name):
+ config = _config(name)
+ assert _is_text_seq2seq_config(config)
+ assert type(config) in transformers.AutoModelForSeq2SeqLM._model_mapping
+
+
+ "name",
+ [
+ "VoxtralConfig",
+ "Qwen2AudioConfig",
+ "GraniteSpeechConfig",
+ "T5Gemma2Config",
+ "LlamaConfig",
+ "Qwen3Config",
+ "Gemma3Config",
+ "WhisperConfig",
+ ],
+)
+def test_other_configs_keep_their_route(name):
+ assert not _is_text_seq2seq_config(_config(name))
+
+
+def test_no_config():
+ assert not _is_text_seq2seq_config(None)
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 2d631f058f3..00e51adfb36 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -112,7 +112,7 @@ from transformers.cache_utils import DynamicCache, Cache
from ..kernels import *
from ..kernels.utils import has_mxfp4_base
from ..tokenizer_utils import *
-from .vision import FastBaseModel
+from .vision import FastBaseModel, _is_text_seq2seq_config
from transformers.models.llama.modeling_llama import (
LlamaAttention,
@@ -2413,6 +2413,17 @@ def _fused_lora_skip_reason(
)
+_DEFAULT_TARGET_MODULES = [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+]
+
+
def _patched_transformers_modules(model_patcher):
patcher_module = sys.modules.get(getattr(model_patcher, "__module__", ""))
if patcher_module is None:
@@ -3477,15 +3488,7 @@ class FastLlamaModel:
def get_peft_model(
model,
r = 16,
- target_modules = [
- "q_proj",
- "k_proj",
- "v_proj",
- "o_proj",
- "gate_proj",
- "up_proj",
- "down_proj",
- ],
+ target_modules = _DEFAULT_TARGET_MODULES,
lora_alpha = 16,
lora_dropout = 0.0,
bias = "none",
@@ -3505,7 +3508,9 @@ class FastLlamaModel:
ensure_weight_tying = None, # None = auto (tie when we redirect a tied pair)
**kwargs,
):
- if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1":
+ # The flag reflects the LAST load, not this model.
+ _text_seq2seq = _is_text_seq2seq_config(getattr(model, "config", None))
+ if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1" or _text_seq2seq:
for peft_arg, flag in (
("finetune_vision_layers", False),
("finetune_language_layers", True),
@@ -3515,6 +3520,9 @@ class FastLlamaModel:
):
if peft_arg not in kwargs:
kwargs[peft_arg] = flag
+ # Identity, not equality: only an omitted argument is replaced; the causal default names no T5 leaf.
+ if target_modules is _DEFAULT_TARGET_MODULES and _text_seq2seq:
+ target_modules = None
return FastBaseModel.get_peft_model(
model = model,
r = r,
@@ -4068,7 +4076,9 @@ class FastLlamaModel:
# module flags every GRPO step, and TrainingArguments defaults it to False, which would silently
# disable it at train time (#4735). Recorded here so loader.py's from_pretrained path is covered.
model._unsloth_gradient_checkpointing = use_gradient_checkpointing
- if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1":
+ if os.environ.get("UNSLOTH_USE_NEW_MODEL", "0") == "1" or _is_text_seq2seq_config(
+ getattr(model, "config", None)
+ ):
return FastBaseModel.patch_peft_model(
model = model,
use_gradient_checkpointing = use_gradient_checkpointing,
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 303e8a3c1d7..bf105bc739a 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -1551,7 +1551,7 @@ from ..kernels import (
patch_loss_functions,
post_patch_loss_function,
)
-from .vision import FastBaseModel
+from .vision import FastBaseModel, _is_text_seq2seq_config
from .diffusion import FastDiffusionModel, is_diffusion_model_type
from transformers import (
AutoModelForCausalLM,
@@ -2138,6 +2138,13 @@ class FastModel(FastBaseModel):
+ NIGHTLY
)
break
+ # transformers 4.x T5: the fullgraph-compiled layer inlines a compiler-disabled T5Attention (dynamo Unsupported).
+ if (
+ transformers_version < Version("5.0.0")
+ and getattr(model_config, "model_type", None) in ("t5", "mt5", "umt5")
+ and _is_text_seq2seq_config(model_config)
+ ):
+ os.environ["UNSLOTH_COMPILE_DISABLE"] = "partial"
if auto_model is not None:
# All other models need to disable static cache.
@@ -2266,7 +2273,8 @@ class FastModel(FastBaseModel):
sdpa_gqa_replace = True,
sdpa_dynamic_compile = True,
compile_attention = True,
- disable_causal_masks = True,
+ # Encoder-decoders on transformers 4.x build the decoder's causal mask in _update_causal_mask; stubbing it makes the decoder bidirectional under eager attention (T5).
+ disable_causal_masks = not _is_text_seq2seq_config(model_config),
compile_torch_modules = True,
compile_custom_modules = True,
compile_function_calls = True,
@@ -2298,6 +2306,8 @@ class FastModel(FastBaseModel):
_ckpt_is_vlm = any(x.endswith("ForConditionalGeneration") for x in _ckpt_arch) or hasattr(
model_config, "vision_config"
)
+ # T5 / BART end in ForConditionalGeneration too but ship a tokenizer, not a processor.
+ _ckpt_is_vlm = _ckpt_is_vlm and not _is_text_seq2seq_config(model_config)
tokenizer_name = _resolve_checkpoint_tokenizer_name(
old_model_name, kwargs, require_processor = _ckpt_is_vlm
)
@@ -2446,6 +2456,22 @@ class FastModel(FastBaseModel):
if _num_labels is not None:
from transformers import AutoModelForSequenceClassification
auto_model = AutoModelForSequenceClassification
+ elif _is_text_seq2seq_config(model_config):
+ if fast_inference:
+ raise NotImplementedError(
+ "Unsloth: fast_inference (vLLM) does not support encoder-decoder models "
+ "such as T5 or BART. Please load with fast_inference = False."
+ )
+ from transformers import AutoModelForSeq2SeqLM
+
+ auto_model = AutoModelForSeq2SeqLM
+ # The zoo's source probe says sdpa where transformers refuses it (T5 on 4.57).
+ if not getattr(
+ resolve_model_class(auto_model, model_config, **_probe_hub_kwargs),
+ "_supports_sdpa",
+ True,
+ ):
+ supports_sdpa = False
elif is_vlm:
# Some repo-code VL models register only a generic auto class (Nemotron-VL uses AutoModelForCausalLM, DeepSeek-OCR AutoModel), so the VLM auto class raises "Unrecognized configuration class". Fall back to what the repo registered, matching the CONCRETE class name, since transformers resolves remote code by that exact name.
_auto_map = getattr(model_config, "auto_map", {}) or {}
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 2f780219b9c..3ef77c66f13 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -66,6 +66,32 @@ def _multimodal_auto_classes():
return classes
+def _is_text_seq2seq_config(config):
+ """T5 / BART / Marian: Seq2SeqLM-mapped, no multimodal class or sub-config (Voxtral, T5Gemma2 are Seq2SeqLM too)."""
+ import transformers
+
+ if config is None or any(
+ hasattr(config, key)
+ for key in ("vision_config", "audio_config", "text_config", "encoder_config")
+ ):
+ return False
+
+ def mapped(auto_class):
+ try:
+ return auto_class is not None and type(config) in auto_class._model_mapping
+ except Exception:
+ return False
+
+ if not mapped(getattr(transformers, "AutoModelForSeq2SeqLM", None)):
+ return False
+ return not any(mapped(auto_class) for auto_class in _multimodal_auto_classes())
+
+
+def _generation_padding_side(config):
+ # BART / Marian encoder positions are absolute, so left padding would shift every real token.
+ return "right" if _is_text_seq2seq_config(config) else "left"
+
+
from ..kernels import (
post_patch_loss_function,
)
@@ -144,7 +170,7 @@ from unsloth_zoo.gradient_checkpointing import (
import torch.utils.checkpoint as torch_checkpoint
import transformers.modeling_utils as hf_modeling_utils
from peft import LoraConfig, TaskType, get_peft_model as _get_peft_model
-from peft import PeftModelForCausalLM
+from peft import PeftModelForCausalLM, PeftModelForSeq2SeqLM
from transformers import set_seed as transformers_set_seed
from unsloth_zoo.peft_utils import (
get_peft_regex,
@@ -1292,7 +1318,14 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
if model_eos_token_id is not None and hasattr(model_eos_token_id, "__iter__"):
model_eos_token_id = model_eos_token_id[0]
- kwargs["pad_token_id"] = kwargs.pop("pad_token_id", model_eos_token_id)
+ # Encoder-decoders keep their own pad (T5: pad 0, EOS 1), used to infer the encoder mask and pad finished rows.
+ default_pad_token_id = model_eos_token_id
+ if (
+ _is_text_seq2seq_config(self.config)
+ and getattr(self.config, "pad_token_id", None) is not None
+ ):
+ default_pad_token_id = self.config.pad_token_id
+ kwargs["pad_token_id"] = kwargs.pop("pad_token_id", default_pad_token_id)
try:
kwargs["pixel_values"] = kwargs["pixel_values"].to(dtype)
@@ -3596,9 +3629,10 @@ class FastBaseModel:
apply_accepts_loss_kwargs_fix(model)
patch_gradient_accumulation_fix(Trainer)
- tokenizer.padding_side = "left"
+ _padding_side = _generation_padding_side(getattr(model, "config", None))
+ tokenizer.padding_side = _padding_side
if hasattr(tokenizer, "tokenizer"):
- tokenizer.tokenizer.padding_side = "left"
+ tokenizer.tokenizer.padding_side = _padding_side
# Audio feature extractors must stay right padded: left (a text setting, forwarded by from_pretrained) shifts Whisper mels and desyncs Gemma 4 audio token counts, crashing on transformers < 5.10.
feature_extractor = getattr(tokenizer, "feature_extractor", None)
if (
@@ -3713,8 +3747,34 @@ class FastBaseModel:
if r <= 0:
raise TypeError(f"Unsloth: Rank of {str(r)} must be larger than 0.")
- if isinstance(model, PeftModelForCausalLM):
+ if isinstance(model, (PeftModelForCausalLM, PeftModelForSeq2SeqLM)):
raise RuntimeError("Unsloth: You already added LoRA adapters to your model!")
+ if _is_text_seq2seq_config(getattr(model, "config", None)):
+ if task_type == TaskType.CAUSAL_LM:
+ task_type = TaskType.SEQ_2_SEQ_LM
+ # No vision tower: FastLanguageModel's finetune_vision_layers=False must not filter the encoder out.
+ finetune_vision_layers = True
+ # get_peft_regex misses T5's q/k/v/o/wi/wo and BART's fc1/fc2, so list the Linear leaves (minus the LM head) ourselves.
+ if (
+ target_modules is None or target_modules == "all-linear"
+ ) and finetune_language_layers:
+ _output = model.get_output_embeddings()
+ _linears = [
+ name
+ for name, module in model.named_modules()
+ if isinstance(module, torch.nn.Linear) and module is not _output
+ ]
+ if finetune_attention_modules and finetune_mlp_modules:
+ target_modules = sorted({name.rsplit(".", 1)[-1] for name in _linears})
+ elif finetune_attention_modules or finetune_mlp_modules:
+ target_modules = [
+ name
+ for name in _linears
+ if bool(re.search(r"attention|attn", name.lower()))
+ == finetune_attention_modules
+ ]
+ # Full paths carry the family choice; get_peft_regex would reject them.
+ finetune_attention_modules = finetune_mlp_modules = True
# Remember whether the CALLER explicitly opted into audio: "all-linear" turns the flag on implicitly below, but an old unsloth_zoo without audio must not fail a plain all-linear run.
_audio_explicitly_requested = bool(finetune_audio_layers)
@@ -4210,7 +4270,9 @@ class FastBaseModel:
if hasattr(m, "training"):
m.training = False
if hasattr(m, "_saved_temp_tokenizer"):
- m._saved_temp_tokenizer.padding_side = "left"
+ m._saved_temp_tokenizer.padding_side = _generation_padding_side(
+ getattr(m, "config", None)
+ )
m._flag_for_generation = True
m = model