Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Padding-free rows for Gemma 4 image and audio SFT (#13385)

Daniel Han6dc9ab02add3
diff --git a/tests/test_mm_packing.py b/tests/test_mm_packing.py
new file mode 100644
index 00000000000..ca038323d16
--- /dev/null
+++ b/tests/test_mm_packing.py
@@ -0,0 +1,172 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""pack_multimodal_documents row layout (CPU only)."""
+
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from unsloth.utils.mm_packing import ( # noqa: E402
+ pack_multimodal_documents,
+ prepare_packed_multimodal_model,
+)
+
+
+def _text(ids):
+ t = torch.tensor([ids])
+ return {
+ "input_ids": t,
+ "attention_mask": torch.ones_like(t),
+ "mm_token_type_ids": torch.zeros_like(t),
+ "labels": t.clone(),
+ }
+
+
+def _image(ids, n_img, patches):
+ d = _text(ids)
+ d["mm_token_type_ids"] = (torch.tensor([ids]) == 9).long()
+ d["pixel_values"] = torch.full((n_img, patches, 4), float(len(ids)))
+ d["image_position_ids"] = torch.ones(n_img, patches, 2, dtype = torch.long)
+ return d
+
+
+def _audio(ids, frames):
+ d = _text(ids)
+ d["input_features"] = torch.ones(1, frames, 3)
+ d["input_features_mask"] = torch.ones(1, frames, dtype = torch.bool)
+ return d
+
+
+def test_layout_positions_lengths_and_boundary_labels():
+ docs = [_text([1, 2, 3]), _image([4, 9, 9, 5], 1, 6), _text([6, 7])]
+ out = pack_multimodal_documents(docs)
+ assert out["input_ids"].tolist() == [[1, 2, 3, 4, 9, 9, 5, 6, 7]]
+ assert out["position_ids"].tolist() == [[0, 1, 2, 0, 1, 2, 3, 0, 1]]
+ assert out["packed_seq_lengths"].tolist() == [3, 4, 2]
+ assert out["packed_seq_lengths"].dtype == torch.int32
+ assert out["labels"].tolist() == [[1, 2, 3, -100, 9, 9, 5, -100, 7]]
+ assert out["mm_token_type_ids"].tolist() == [[0, 0, 0, 0, 1, 1, 0, 0, 0]]
+ assert "attention_mask" not in out
+ assert out["pixel_values"].shape == (1, 6, 4)
+
+
+def test_media_concatenate_in_document_order_and_pad_with_the_model_padding():
+ docs = [_image([9, 1], 2, 6), _text([2]), _image([9, 9, 3], 1, 8)]
+ out = pack_multimodal_documents(docs)
+ pv, pos = out["pixel_values"], out["image_position_ids"]
+ assert pv.shape == (3, 8, 4) and pos.shape == (3, 8, 2)
+ assert pv[:2, :6].eq(2.0).all() and pv[2].eq(3.0).all()
+ assert pv[:2, 6:].eq(0).all()
+ assert pos[:2, 6:].eq(-1).all() and pos[:2, :6].eq(1).all() # (-1, -1) = padding patch
+
+
+def test_audio_frames_pad_with_a_false_mask():
+ out = pack_multimodal_documents([_audio([1, 2], 5), _audio([3], 7)])
+ assert out["input_features"].shape == (2, 7, 3)
+ assert out["input_features_mask"].tolist() == [[True] * 5 + [False] * 2, [True] * 7]
+
+
+def test_padding_inside_a_document_is_stripped():
+ d = _text([1, 2, 0, 0])
+ d["attention_mask"] = torch.tensor([[1, 1, 0, 0]])
+ out = pack_multimodal_documents([d, _text([5])])
+ assert out["input_ids"].tolist() == [[1, 2, 5]]
+ assert out["packed_seq_lengths"].tolist() == [2, 1]
+
+
[email protected]("key", ["pixel_values_videos", "image_grid_thw", "pixel_attention_mask"])
+def test_unproven_media_keys_are_refused(key):
+ d = _text([1])
+ d[key] = torch.ones(1, 2)
+ with pytest.raises(ValueError, match = "cannot pack"):
+ pack_multimodal_documents([d])
+
+
+def test_unsupported_models_are_refused():
+ qwen = SimpleNamespace(config = SimpleNamespace(model_type = "qwen2_5_vl"))
+ with pytest.raises(ValueError, match = "only supported"):
+ prepare_packed_multimodal_model(qwen)
+
+
+if __name__ == "__main__":
+ raise SystemExit(pytest.main([__file__, "-q"]))
+
+
+def _collator(padding_free = True, **slots):
+ from unsloth.trainer import UnslothVisionDataCollator
+
+ collator = object.__new__(UnslothVisionDataCollator)
+ collator._padding_free = padding_free
+ collator._deferring_check = False
+ collator._checked_video_paths = set()
+ for name, value in slots.items():
+ setattr(collator, name, value)
+ return collator
+
+
+def test_supervision_is_judged_over_the_batch_not_per_document(monkeypatch):
+ from unsloth.trainer import UnslothVisionDataCollator, _UnslothVisionDataCollatorBase
+
+ if not hasattr(_UnslothVisionDataCollatorBase, "_check_supervised"):
+ pytest.skip("this unsloth_zoo has no supervision check")
+ empty, trained = _text([1, 2, 3]), _text([4, 5])
+ empty["labels"][:] = -100
+ docs = iter([empty, trained])
+ seen = []
+
+ def collate(self, examples):
+ doc = next(docs)
+ self._check_supervised(doc["labels"], doc["attention_mask"])
+ return doc
+
+ monkeypatch.setattr(UnslothVisionDataCollator, "_collate_unpacked", collate)
+ monkeypatch.setattr(
+ _UnslothVisionDataCollatorBase,
+ "_check_supervised",
+ lambda self, labels, attention_mask = None: seen.append(labels.clone()),
+ )
+ collator = _collator(ignore_index = -100, train_on_responses_only = lambda batch: batch)
+ out = collator([{}, {}])
+ # One check, over both documents: the empty first one alone would read as an unsupervised batch.
+ assert len(seen) == 1 and seen[0].shape == (2, 3)
+ assert seen[0][0].eq(-100).all() and seen[0][1, :2].tolist() == [4, 5]
+ assert out["packed_seq_lengths"].tolist() == [3, 2]
+
+
+def _trainer(cp_enabled = False):
+ return SimpleNamespace(
+ accelerator = SimpleNamespace(parallelism_config = SimpleNamespace(cp_enabled = cp_enabled))
+ )
+
+
+def test_packed_collation_yields_to_eval_logits_and_context_parallelism(monkeypatch):
+ from unsloth.trainer import _collator_packing_conflict
+
+ # The trainer's own for_inference() leaves this set after __init__; only the pre-init read counts.
+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")
+ assert _collator_packing_conflict(_trainer(), False) is None
+ assert "prediction row" in _collator_packing_conflict(_trainer(), True)
+ assert "context parallelism" in _collator_packing_conflict(_trainer(cp_enabled = True), False)
+ assert _collator_packing_conflict(SimpleNamespace(), False) is None
+
+
+def test_a_forward_without_packed_seq_lengths_turns_the_collator_off(monkeypatch):
+ from unsloth.trainer import _collator_packing_conflict
+ assert "packed_seq_lengths" in _collator_packing_conflict(_trainer(), False, True)
+
+
[email protected]("getter", ["get_eval_dataloader", "get_test_dataloader"])
+def test_evaluation_dataloaders_are_built_from_an_unpacked_collator(getter):
+ from unsloth.trainer import _unpacked_for_evaluation
+
+ collator = _collator()
+ trainer = SimpleNamespace(data_collator = collator)
+ trainer.get_eval_dataloader = lambda dataset = None: trainer.data_collator
+ trainer.get_test_dataloader = lambda dataset: trainer.data_collator
+ _unpacked_for_evaluation(trainer, collator)
+ used = getattr(trainer, getter)([])
+ assert used is not collator and used.padding_free is False
+ assert trainer.data_collator is collator and collator.padding_free is True
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 0e4dfd42cc9..43bdcd6ebe6 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -66,13 +66,51 @@ class UnslothVisionDataCollator(_UnslothVisionDataCollatorBase):
of silently training on empty video tensors (issue #5085).
"""
- __slots__ = ("_checked_video_paths",)
+ __slots__ = ("_checked_video_paths", "_padding_free", "_deferring_check")
- def __init__(self, *args, **kwargs):
+ def __init__(
+ self,
+ *args,
+ padding_free = False,
+ **kwargs,
+ ):
super().__init__(*args, **kwargs)
self._checked_video_paths = set()
+ self._padding_free = bool(padding_free)
+ if self._padding_free:
+ from unsloth.utils.mm_packing import prepare_packed_multimodal_model
+ prepare_packed_multimodal_model(args[0] if args else kwargs.get("model"))
+
+ @property
+ def padding_free(self):
+ return getattr(self, "_padding_free", False)
def __call__(self, examples):
+ if getattr(self, "_padding_free", False):
+ from unsloth.utils.mm_packing import pack_multimodal_documents
+
+ # Collate one document at a time, but judge supervision over the whole batch as the base does.
+ self._deferring_check = True
+ try:
+ docs = [self._collate_unpacked([example]) for example in examples]
+ finally:
+ self._deferring_check = False
+ if self.train_on_responses_only and hasattr(
+ _UnslothVisionDataCollatorBase, "_check_supervised"
+ ):
+ self._check_supervised(*_stack_rows(docs))
+ return pack_multimodal_documents(docs, ignore_index = self.ignore_index)
+ return self._collate_unpacked(examples)
+
+ def _check_supervised(
+ self,
+ labels,
+ attention_mask = None,
+ ):
+ if not getattr(self, "_deferring_check", False):
+ return super()._check_supervised(labels, attention_mask)
+
+ def _collate_unpacked(self, examples):
formatting_func = self.formatting_func
if formatting_func is not None:
examples = [formatting_func(example) for example in examples]
@@ -92,6 +130,58 @@ class UnslothVisionDataCollator(_UnslothVisionDataCollatorBase):
return super(UnslothVisionDataCollator, view).__call__(examples)
+def _stack_rows(docs):
+ """Per-document labels / attention masks as one right-padded batch."""
+ import torch
+
+ width = max(doc["labels"].shape[-1] for doc in docs)
+ labels = torch.full((len(docs), width), -100, dtype = docs[0]["labels"].dtype)
+ attention_mask = torch.zeros((len(docs), width), dtype = torch.long)
+ for row, doc in enumerate(docs):
+ n = doc["labels"].shape[-1]
+ labels[row, :n] = doc["labels"][0].cpu()
+ mask = doc.get("attention_mask")
+ attention_mask[row, :n] = mask[0].cpu() if torch.is_tensor(mask) else 1
+ return labels, attention_mask
+
+
+def _collator_packing_conflict(
+ trainer,
+ wants_eval_logits,
+ forward_rejects_packing = False,
+):
+ """Why a packing collator must fall back to padded rows, else None (the args-level gates miss it)."""
+ if forward_rejects_packing:
+ return "the model's forward() does not accept packed_seq_lengths"
+ if wants_eval_logits:
+ return "evaluation needs one prediction row per example"
+ parallelism_config = getattr(getattr(trainer, "accelerator", None), "parallelism_config", None)
+ if getattr(parallelism_config, "cp_enabled", False):
+ return "context parallelism drops the per-document attention mask"
+ return None
+
+
+def _unpacked_for_evaluation(trainer, collator):
+ """Pack training batches only: evaluation counts examples (and weights eval_loss) by input_ids.shape[0].
+ A DataLoader captures its collate_fn when built, so build eval / test ones from an unpacked copy."""
+ unpacked = copy.copy(collator)
+ unpacked._padding_free = False
+
+ def unpack(original):
+ @wraps(original)
+ def get_dataloader(*args, **kwargs):
+ packed, trainer.data_collator = trainer.data_collator, unpacked
+ try:
+ return original(*args, **kwargs)
+ finally:
+ trainer.data_collator = packed
+
+ return get_dataloader
+
+ trainer.get_eval_dataloader = unpack(trainer.get_eval_dataloader)
+ trainer.get_test_dataloader = unpack(trainer.get_test_dataloader)
+
+
_AUTO_PADDING_FREE_ENV_DISABLED = os.environ.get(
"UNSLOTH_DISABLE_AUTO_PADDING_FREE", ""
).strip().lower() in {"1", "true", "yes", "on"}
@@ -1752,6 +1842,10 @@ def _patch_sft_trainer_auto_packing(trl_module):
# several classes, so a "yes" from the resolved one is not proof about the
# instance. A correct "no" has already turned both flags off, which is the
# condition the post-init check skips on, so leaving it armed is free.
+ # Read before __init__: the compiled trainer ends it with model.for_inference(), which sets the flag.
+ eval_logits_wanted = os.environ.get(
+ "UNSLOTH_RETURN_LOGITS", "0"
+ ) == "1" or _wants_eval_logits(original_init, args, kwargs)
blocked = (
(data_collator is not None)
or (is_processor and not processor_text_only)
@@ -1760,8 +1854,7 @@ def _patch_sft_trainer_auto_packing(trl_module):
or is_unsupported_model
or is_encoder_decoder
or (is_hybrid and not hybrid_varlen_active)
- or (os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1")
- or _wants_eval_logits(original_init, args, kwargs)
+ or eval_logits_wanted
or forward_rejects_packing
)
requested_pack = bool(getattr(config_arg, "packing", False))
@@ -1889,6 +1982,17 @@ def _patch_sft_trainer_auto_packing(trl_module):
setattr(trainer_args, "packing", False)
setattr(trainer_args, "padding_free", False)
+ collator = getattr(self, "data_collator", None)
+ if getattr(collator, "padding_free", False):
+ reason = _collator_packing_conflict(self, eval_logits_wanted, forward_rejects_packing)
+ if reason is not None:
+ collator._padding_free = False
+ logger.warning(
+ f"Unsloth: UnslothVisionDataCollator(padding_free=True) ignored ({reason})."
+ )
+ else:
+ _unpacked_for_evaluation(self, collator)
+
if not blocked and trainer_packing and (packing_active or _should_pack(trainer_args)):
enable_sample_packing(self.model, self)
print("🦥 Unsloth: Packing enabled - training is >2x faster and uses less VRAM!")
diff --git a/unsloth/utils/mm_packing.py b/unsloth/utils/mm_packing.py
new file mode 100644
index 00000000000..689dfa426bc
--- /dev/null
+++ b/unsloth/utils/mm_packing.py
@@ -0,0 +1,135 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Packed rows of multimodal documents: each collated alone, then concatenated with per-document
+``position_ids``, from which transformers masks image blocks to their own document."""
+
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+import torch
+
+__all__ = [
+ "pack_multimodal_documents",
+ "prepare_packed_multimodal_model",
+ "SUPPORTED_PACKED_MULTIMODAL_MODEL_TYPES",
+]
+
+SUPPORTED_PACKED_MULTIMODAL_MODEL_TYPES = frozenset({"gemma4"})
+
+_TOKEN_KEYS = ("input_ids", "labels", "mm_token_type_ids", "token_type_ids")
+# Media key -> value padding a mismatched trailing axis (the towers mask these out).
+_MEDIA_PAD = {
+ "pixel_values": 0.0,
+ "image_position_ids": -1,
+ "pixel_position_ids": -1,
+ "input_features": 0.0,
+ "input_features_mask": False,
+}
+_DROPPED = ("attention_mask", "position_ids", "packed_seq_lengths")
+
+
+def _pad_to(t: torch.Tensor, shape: Sequence[int], value) -> torch.Tensor:
+ if tuple(t.shape[1:]) == tuple(shape):
+ return t
+ out = t.new_full((t.shape[0], *shape), value)
+ out[(slice(None), *(slice(0, n) for n in t.shape[1:]))] = t
+ return out
+
+
+def _strip_padding(doc: dict) -> dict:
+ # Batch-of-one can still be padded (pad_to_multiple_of).
+ am = doc.get("attention_mask")
+ if am is None or bool(am.all()):
+ return doc
+ keep = am[0].bool()
+ doc = dict(doc)
+ for key in _TOKEN_KEYS:
+ if key in doc and torch.is_tensor(doc[key]) and doc[key].dim() == 2:
+ doc[key] = doc[key][:, keep]
+ return doc
+
+
+def pack_multimodal_documents(docs: Sequence[dict], ignore_index: int = -100) -> dict:
+ if not docs:
+ raise ValueError("Unsloth: no documents to pack")
+ docs = [_strip_padding(d) for d in docs]
+ for d in docs:
+ if d["input_ids"].shape[0] != 1:
+ raise ValueError("Unsloth: pack_multimodal_documents expects batch-of-one documents")
+ unknown = (
+ set().union(*(d.keys() for d in docs)) - set(_TOKEN_KEYS) - set(_MEDIA_PAD) - set(_DROPPED)
+ )
+ if unknown:
+ # Unproven keys: refuse rather than drop silently.
+ raise ValueError(f"Unsloth: cannot pack multimodal key(s) {sorted(unknown)}")
+
+ lengths = [int(d["input_ids"].shape[1]) for d in docs]
+ out: dict[str, Any] = {}
+ for key in _TOKEN_KEYS:
+ present = [key in d for d in docs]
+ if not any(present):
+ continue
+ if not all(present):
+ if key == "labels":
+ raise ValueError("Unsloth: some documents have labels and some do not")
+ parts = [d[key] if key in d else torch.zeros_like(d["input_ids"]) for d in docs]
+ else:
+ parts = [d[key] for d in docs]
+ out[key] = torch.cat(parts, dim = 1)
+
+ if "labels" in out:
+ labels = out["labels"].clone()
+ start = 0
+ for n in lengths[:-1]:
+ start += n
+ labels[0, start] = ignore_index
+ out["labels"] = labels
+
+ device = out["input_ids"].device
+ out["position_ids"] = torch.cat([torch.arange(n, device = device) for n in lengths])[None]
+ out["packed_seq_lengths"] = torch.tensor(lengths, dtype = torch.int32)
+
+ for key, pad in _MEDIA_PAD.items():
+ parts = [d[key] for d in docs if key in d and d[key] is not None]
+ if not parts:
+ continue
+ trailing = [max(p.shape[i] for p in parts) for i in range(1, parts[0].dim())]
+ if any(p.dim() != parts[0].dim() for p in parts):
+ raise ValueError(f"Unsloth: {key} rank differs between documents")
+ out[key] = torch.cat([_pad_to(p, trailing, pad) for p in parts], dim = 0)
+ if "input_features" in out and "input_features_mask" not in out:
+ raise ValueError("Unsloth: input_features without input_features_mask cannot be packed")
+ return out
+
+
+def _model_types(config) -> set:
+ types = set()
+ stack = [config]
+ while stack:
+ c = stack.pop()
+ if c is None:
+ continue
+ mt = getattr(c, "model_type", None)
+ if mt:
+ types.add(mt)
+ for sub in ("text_config", "vision_config", "audio_config"):
+ s = getattr(c, sub, None)
+ if s is not None and s is not c:
+ stack.append(s)
+ return types
+
+
+def prepare_packed_multimodal_model(model) -> None:
+ types = _model_types(getattr(model, "config", None))
+ if not types & SUPPORTED_PACKED_MULTIMODAL_MODEL_TYPES:
+ raise ValueError(
+ "Unsloth: padding_free=True in UnslothVisionDataCollator is only supported for "
+ f"{sorted(SUPPORTED_PACKED_MULTIMODAL_MODEL_TYPES)} for now, got "
+ f"{sorted(types) or 'an unknown model'}. Set padding_free=False."
+ )
+ from .packing import _enable_hf_packed_attention, mark_allow_overlength
+
+ _enable_hf_packed_attention()
+ mark_allow_overlength(model)