Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Padding-free rows for Gemma 4 image and audio SFT (#13385)
Daniel Han6dc9ab02add3
diff --git a/tests/test_mm_packing.py b/tests/test_mm_packing.pynew file mode 100644index 00000000000..ca038323d16--- /dev/null+++ b/tests/test_mm_packing.py@@ -0,0 +1,172 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""pack_multimodal_documents row layout (CPU only)."""++from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")++from unsloth.utils.mm_packing import ( # noqa: E402+ pack_multimodal_documents,+ prepare_packed_multimodal_model,+)+++def _text(ids):+ t = torch.tensor([ids])+ return {+ "input_ids": t,+ "attention_mask": torch.ones_like(t),+ "mm_token_type_ids": torch.zeros_like(t),+ "labels": t.clone(),+ }+++def _image(ids, n_img, patches):+ d = _text(ids)+ d["mm_token_type_ids"] = (torch.tensor([ids]) == 9).long()+ d["pixel_values"] = torch.full((n_img, patches, 4), float(len(ids)))+ d["image_position_ids"] = torch.ones(n_img, patches, 2, dtype = torch.long)+ return d+++def _audio(ids, frames):+ d = _text(ids)+ d["input_features"] = torch.ones(1, frames, 3)+ d["input_features_mask"] = torch.ones(1, frames, dtype = torch.bool)+ return d+++def test_layout_positions_lengths_and_boundary_labels():+ docs = [_text([1, 2, 3]), _image([4, 9, 9, 5], 1, 6), _text([6, 7])]+ out = pack_multimodal_documents(docs)+ assert out["input_ids"].tolist() == [[1, 2, 3, 4, 9, 9, 5, 6, 7]]+ assert out["position_ids"].tolist() == [[0, 1, 2, 0, 1, 2, 3, 0, 1]]+ assert out["packed_seq_lengths"].tolist() == [3, 4, 2]+ assert out["packed_seq_lengths"].dtype == torch.int32+ assert out["labels"].tolist() == [[1, 2, 3, -100, 9, 9, 5, -100, 7]]+ assert out["mm_token_type_ids"].tolist() == [[0, 0, 0, 0, 1, 1, 0, 0, 0]]+ assert "attention_mask" not in out+ assert out["pixel_values"].shape == (1, 6, 4)+++def test_media_concatenate_in_document_order_and_pad_with_the_model_padding():+ docs = [_image([9, 1], 2, 6), _text([2]), _image([9, 9, 3], 1, 8)]+ out = pack_multimodal_documents(docs)+ pv, pos = out["pixel_values"], out["image_position_ids"]+ assert pv.shape == (3, 8, 4) and pos.shape == (3, 8, 2)+ assert pv[:2, :6].eq(2.0).all() and pv[2].eq(3.0).all()+ assert pv[:2, 6:].eq(0).all()+ assert pos[:2, 6:].eq(-1).all() and pos[:2, :6].eq(1).all() # (-1, -1) = padding patch+++def test_audio_frames_pad_with_a_false_mask():+ out = pack_multimodal_documents([_audio([1, 2], 5), _audio([3], 7)])+ assert out["input_features"].shape == (2, 7, 3)+ assert out["input_features_mask"].tolist() == [[True] * 5 + [False] * 2, [True] * 7]+++def test_padding_inside_a_document_is_stripped():+ d = _text([1, 2, 0, 0])+ d["attention_mask"] = torch.tensor([[1, 1, 0, 0]])+ out = pack_multimodal_documents([d, _text([5])])+ assert out["input_ids"].tolist() == [[1, 2, 5]]+ assert out["packed_seq_lengths"].tolist() == [2, 1]++[email protected]("key", ["pixel_values_videos", "image_grid_thw", "pixel_attention_mask"])+def test_unproven_media_keys_are_refused(key):+ d = _text([1])+ d[key] = torch.ones(1, 2)+ with pytest.raises(ValueError, match = "cannot pack"):+ pack_multimodal_documents([d])+++def test_unsupported_models_are_refused():+ qwen = SimpleNamespace(config = SimpleNamespace(model_type = "qwen2_5_vl"))+ with pytest.raises(ValueError, match = "only supported"):+ prepare_packed_multimodal_model(qwen)+++if __name__ == "__main__":+ raise SystemExit(pytest.main([__file__, "-q"]))+++def _collator(padding_free = True, **slots):+ from unsloth.trainer import UnslothVisionDataCollator++ collator = object.__new__(UnslothVisionDataCollator)+ collator._padding_free = padding_free+ collator._deferring_check = False+ collator._checked_video_paths = set()+ for name, value in slots.items():+ setattr(collator, name, value)+ return collator+++def test_supervision_is_judged_over_the_batch_not_per_document(monkeypatch):+ from unsloth.trainer import UnslothVisionDataCollator, _UnslothVisionDataCollatorBase++ if not hasattr(_UnslothVisionDataCollatorBase, "_check_supervised"):+ pytest.skip("this unsloth_zoo has no supervision check")+ empty, trained = _text([1, 2, 3]), _text([4, 5])+ empty["labels"][:] = -100+ docs = iter([empty, trained])+ seen = []++ def collate(self, examples):+ doc = next(docs)+ self._check_supervised(doc["labels"], doc["attention_mask"])+ return doc++ monkeypatch.setattr(UnslothVisionDataCollator, "_collate_unpacked", collate)+ monkeypatch.setattr(+ _UnslothVisionDataCollatorBase,+ "_check_supervised",+ lambda self, labels, attention_mask = None: seen.append(labels.clone()),+ )+ collator = _collator(ignore_index = -100, train_on_responses_only = lambda batch: batch)+ out = collator([{}, {}])+ # One check, over both documents: the empty first one alone would read as an unsupervised batch.+ assert len(seen) == 1 and seen[0].shape == (2, 3)+ assert seen[0][0].eq(-100).all() and seen[0][1, :2].tolist() == [4, 5]+ assert out["packed_seq_lengths"].tolist() == [3, 2]+++def _trainer(cp_enabled = False):+ return SimpleNamespace(+ accelerator = SimpleNamespace(parallelism_config = SimpleNamespace(cp_enabled = cp_enabled))+ )+++def test_packed_collation_yields_to_eval_logits_and_context_parallelism(monkeypatch):+ from unsloth.trainer import _collator_packing_conflict++ # The trainer's own for_inference() leaves this set after __init__; only the pre-init read counts.+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")+ assert _collator_packing_conflict(_trainer(), False) is None+ assert "prediction row" in _collator_packing_conflict(_trainer(), True)+ assert "context parallelism" in _collator_packing_conflict(_trainer(cp_enabled = True), False)+ assert _collator_packing_conflict(SimpleNamespace(), False) is None+++def test_a_forward_without_packed_seq_lengths_turns_the_collator_off(monkeypatch):+ from unsloth.trainer import _collator_packing_conflict+ assert "packed_seq_lengths" in _collator_packing_conflict(_trainer(), False, True)++[email protected]("getter", ["get_eval_dataloader", "get_test_dataloader"])+def test_evaluation_dataloaders_are_built_from_an_unpacked_collator(getter):+ from unsloth.trainer import _unpacked_for_evaluation++ collator = _collator()+ trainer = SimpleNamespace(data_collator = collator)+ trainer.get_eval_dataloader = lambda dataset = None: trainer.data_collator+ trainer.get_test_dataloader = lambda dataset: trainer.data_collator+ _unpacked_for_evaluation(trainer, collator)+ used = getattr(trainer, getter)([])+ assert used is not collator and used.padding_free is False+ assert trainer.data_collator is collator and collator.padding_free is Truediff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 0e4dfd42cc9..43bdcd6ebe6 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -66,13 +66,51 @@ class UnslothVisionDataCollator(_UnslothVisionDataCollatorBase):of silently training on empty video tensors (issue #5085)."""- __slots__ = ("_checked_video_paths",)+ __slots__ = ("_checked_video_paths", "_padding_free", "_deferring_check")- def __init__(self, *args, **kwargs):+ def __init__(+ self,+ *args,+ padding_free = False,+ **kwargs,+ ):super().__init__(*args, **kwargs)self._checked_video_paths = set()+ self._padding_free = bool(padding_free)+ if self._padding_free:+ from unsloth.utils.mm_packing import prepare_packed_multimodal_model+ prepare_packed_multimodal_model(args[0] if args else kwargs.get("model"))++ @property+ def padding_free(self):+ return getattr(self, "_padding_free", False)def __call__(self, examples):+ if getattr(self, "_padding_free", False):+ from unsloth.utils.mm_packing import pack_multimodal_documents++ # Collate one document at a time, but judge supervision over the whole batch as the base does.+ self._deferring_check = True+ try:+ docs = [self._collate_unpacked([example]) for example in examples]+ finally:+ self._deferring_check = False+ if self.train_on_responses_only and hasattr(+ _UnslothVisionDataCollatorBase, "_check_supervised"+ ):+ self._check_supervised(*_stack_rows(docs))+ return pack_multimodal_documents(docs, ignore_index = self.ignore_index)+ return self._collate_unpacked(examples)++ def _check_supervised(+ self,+ labels,+ attention_mask = None,+ ):+ if not getattr(self, "_deferring_check", False):+ return super()._check_supervised(labels, attention_mask)++ def _collate_unpacked(self, examples):formatting_func = self.formatting_funcif formatting_func is not None:examples = [formatting_func(example) for example in examples]@@ -92,6 +130,58 @@ class UnslothVisionDataCollator(_UnslothVisionDataCollatorBase):return super(UnslothVisionDataCollator, view).__call__(examples)+def _stack_rows(docs):+ """Per-document labels / attention masks as one right-padded batch."""+ import torch++ width = max(doc["labels"].shape[-1] for doc in docs)+ labels = torch.full((len(docs), width), -100, dtype = docs[0]["labels"].dtype)+ attention_mask = torch.zeros((len(docs), width), dtype = torch.long)+ for row, doc in enumerate(docs):+ n = doc["labels"].shape[-1]+ labels[row, :n] = doc["labels"][0].cpu()+ mask = doc.get("attention_mask")+ attention_mask[row, :n] = mask[0].cpu() if torch.is_tensor(mask) else 1+ return labels, attention_mask+++def _collator_packing_conflict(+ trainer,+ wants_eval_logits,+ forward_rejects_packing = False,+):+ """Why a packing collator must fall back to padded rows, else None (the args-level gates miss it)."""+ if forward_rejects_packing:+ return "the model's forward() does not accept packed_seq_lengths"+ if wants_eval_logits:+ return "evaluation needs one prediction row per example"+ parallelism_config = getattr(getattr(trainer, "accelerator", None), "parallelism_config", None)+ if getattr(parallelism_config, "cp_enabled", False):+ return "context parallelism drops the per-document attention mask"+ return None+++def _unpacked_for_evaluation(trainer, collator):+ """Pack training batches only: evaluation counts examples (and weights eval_loss) by input_ids.shape[0].+ A DataLoader captures its collate_fn when built, so build eval / test ones from an unpacked copy."""+ unpacked = copy.copy(collator)+ unpacked._padding_free = False++ def unpack(original):+ @wraps(original)+ def get_dataloader(*args, **kwargs):+ packed, trainer.data_collator = trainer.data_collator, unpacked+ try:+ return original(*args, **kwargs)+ finally:+ trainer.data_collator = packed++ return get_dataloader++ trainer.get_eval_dataloader = unpack(trainer.get_eval_dataloader)+ trainer.get_test_dataloader = unpack(trainer.get_test_dataloader)++_AUTO_PADDING_FREE_ENV_DISABLED = os.environ.get("UNSLOTH_DISABLE_AUTO_PADDING_FREE", "").strip().lower() in {"1", "true", "yes", "on"}@@ -1752,6 +1842,10 @@ def _patch_sft_trainer_auto_packing(trl_module):# several classes, so a "yes" from the resolved one is not proof about the# instance. A correct "no" has already turned both flags off, which is the# condition the post-init check skips on, so leaving it armed is free.+ # Read before __init__: the compiled trainer ends it with model.for_inference(), which sets the flag.+ eval_logits_wanted = os.environ.get(+ "UNSLOTH_RETURN_LOGITS", "0"+ ) == "1" or _wants_eval_logits(original_init, args, kwargs)blocked = ((data_collator is not None)or (is_processor and not processor_text_only)@@ -1760,8 +1854,7 @@ def _patch_sft_trainer_auto_packing(trl_module):or is_unsupported_modelor is_encoder_decoderor (is_hybrid and not hybrid_varlen_active)- or (os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1")- or _wants_eval_logits(original_init, args, kwargs)+ or eval_logits_wantedor forward_rejects_packing)requested_pack = bool(getattr(config_arg, "packing", False))@@ -1889,6 +1982,17 @@ def _patch_sft_trainer_auto_packing(trl_module):setattr(trainer_args, "packing", False)setattr(trainer_args, "padding_free", False)+ collator = getattr(self, "data_collator", None)+ if getattr(collator, "padding_free", False):+ reason = _collator_packing_conflict(self, eval_logits_wanted, forward_rejects_packing)+ if reason is not None:+ collator._padding_free = False+ logger.warning(+ f"Unsloth: UnslothVisionDataCollator(padding_free=True) ignored ({reason})."+ )+ else:+ _unpacked_for_evaluation(self, collator)+if not blocked and trainer_packing and (packing_active or _should_pack(trainer_args)):enable_sample_packing(self.model, self)print("🦥 Unsloth: Packing enabled - training is >2x faster and uses less VRAM!")diff --git a/unsloth/utils/mm_packing.py b/unsloth/utils/mm_packing.pynew file mode 100644index 00000000000..689dfa426bc--- /dev/null+++ b/unsloth/utils/mm_packing.py@@ -0,0 +1,135 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Packed rows of multimodal documents: each collated alone, then concatenated with per-document+``position_ids``, from which transformers masks image blocks to their own document."""++from __future__ import annotations++from typing import Any, Sequence++import torch++__all__ = [+ "pack_multimodal_documents",+ "prepare_packed_multimodal_model",+ "SUPPORTED_PACKED_MULTIMODAL_MODEL_TYPES",+]++SUPPORTED_PACKED_MULTIMODAL_MODEL_TYPES = frozenset({"gemma4"})++_TOKEN_KEYS = ("input_ids", "labels", "mm_token_type_ids", "token_type_ids")+# Media key -> value padding a mismatched trailing axis (the towers mask these out).+_MEDIA_PAD = {+ "pixel_values": 0.0,+ "image_position_ids": -1,+ "pixel_position_ids": -1,+ "input_features": 0.0,+ "input_features_mask": False,+}+_DROPPED = ("attention_mask", "position_ids", "packed_seq_lengths")+++def _pad_to(t: torch.Tensor, shape: Sequence[int], value) -> torch.Tensor:+ if tuple(t.shape[1:]) == tuple(shape):+ return t+ out = t.new_full((t.shape[0], *shape), value)+ out[(slice(None), *(slice(0, n) for n in t.shape[1:]))] = t+ return out+++def _strip_padding(doc: dict) -> dict:+ # Batch-of-one can still be padded (pad_to_multiple_of).+ am = doc.get("attention_mask")+ if am is None or bool(am.all()):+ return doc+ keep = am[0].bool()+ doc = dict(doc)+ for key in _TOKEN_KEYS:+ if key in doc and torch.is_tensor(doc[key]) and doc[key].dim() == 2:+ doc[key] = doc[key][:, keep]+ return doc+++def pack_multimodal_documents(docs: Sequence[dict], ignore_index: int = -100) -> dict:+ if not docs:+ raise ValueError("Unsloth: no documents to pack")+ docs = [_strip_padding(d) for d in docs]+ for d in docs:+ if d["input_ids"].shape[0] != 1:+ raise ValueError("Unsloth: pack_multimodal_documents expects batch-of-one documents")+ unknown = (+ set().union(*(d.keys() for d in docs)) - set(_TOKEN_KEYS) - set(_MEDIA_PAD) - set(_DROPPED)+ )+ if unknown:+ # Unproven keys: refuse rather than drop silently.+ raise ValueError(f"Unsloth: cannot pack multimodal key(s) {sorted(unknown)}")++ lengths = [int(d["input_ids"].shape[1]) for d in docs]+ out: dict[str, Any] = {}+ for key in _TOKEN_KEYS:+ present = [key in d for d in docs]+ if not any(present):+ continue+ if not all(present):+ if key == "labels":+ raise ValueError("Unsloth: some documents have labels and some do not")+ parts = [d[key] if key in d else torch.zeros_like(d["input_ids"]) for d in docs]+ else:+ parts = [d[key] for d in docs]+ out[key] = torch.cat(parts, dim = 1)++ if "labels" in out:+ labels = out["labels"].clone()+ start = 0+ for n in lengths[:-1]:+ start += n+ labels[0, start] = ignore_index+ out["labels"] = labels++ device = out["input_ids"].device+ out["position_ids"] = torch.cat([torch.arange(n, device = device) for n in lengths])[None]+ out["packed_seq_lengths"] = torch.tensor(lengths, dtype = torch.int32)++ for key, pad in _MEDIA_PAD.items():+ parts = [d[key] for d in docs if key in d and d[key] is not None]+ if not parts:+ continue+ trailing = [max(p.shape[i] for p in parts) for i in range(1, parts[0].dim())]+ if any(p.dim() != parts[0].dim() for p in parts):+ raise ValueError(f"Unsloth: {key} rank differs between documents")+ out[key] = torch.cat([_pad_to(p, trailing, pad) for p in parts], dim = 0)+ if "input_features" in out and "input_features_mask" not in out:+ raise ValueError("Unsloth: input_features without input_features_mask cannot be packed")+ return out+++def _model_types(config) -> set:+ types = set()+ stack = [config]+ while stack:+ c = stack.pop()+ if c is None:+ continue+ mt = getattr(c, "model_type", None)+ if mt:+ types.add(mt)+ for sub in ("text_config", "vision_config", "audio_config"):+ s = getattr(c, sub, None)+ if s is not None and s is not c:+ stack.append(s)+ return types+++def prepare_packed_multimodal_model(model) -> None:+ types = _model_types(getattr(model, "config", None))+ if not types & SUPPORTED_PACKED_MULTIMODAL_MODEL_TYPES:+ raise ValueError(+ "Unsloth: padding_free=True in UnslothVisionDataCollator is only supported for "+ f"{sorted(SUPPORTED_PACKED_MULTIMODAL_MODEL_TYPES)} for now, got "+ f"{sorted(types) or 'an unknown model'}. Set padding_free=False."+ )+ from .packing import _enable_hf_packed_attention, mark_allow_overlength++ _enable_hf_packed_attention()+ mark_allow_overlength(model)