Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Mask packed document boundary labels for TRL older than 0.23.1 (#13324)
Daniel Han7aa07618b964
diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.pyindex 807c00ff0e9..26c30a0bcc3 100644--- a/tests/utils/test_packing.py+++ b/tests/utils/test_packing.py@@ -2301,11 +2301,9 @@ def test_fused_ce_branch_masks_packed_boundaries(monkeypatch, module_name):assert labels.reshape(-1).tolist() == list(range(seq))-# 3. the collator wrappers must leave boundary targets in place: unsloth_zoo counts num_items_in_batch off this-# batch and already deducts them+# wrappers match TRL >= 0.23.1 masking; the zoo count includes each real target onceclass _UnmaskedPackingCollator:- """Padding-free collator that does NOT pre-mask boundaries, like TRL < 0.24 - a test- built on TRL 0.24+ output would pass either way."""+ """padding-free collator with unmasked boundaries, like TRL < 0.23.1."""def __init__(self):self.padding_free = True@@ -2320,16 +2318,23 @@ class _UnmaskedPackingCollator:def _zoo_num_items_in_batch(batch):- """The count unsloth_zoo._unsloth_get_batch_samples derives from a batch."""- count = int((batch["labels"][..., 1:] != -100).sum())+ """unsloth_zoo._unsloth_get_batch_samples' count: boundary slots zeroed, not subtracted."""+ token_count = batch["labels"][..., 1:] != -100lengths = batch.get("packed_seq_lengths")if lengths is not None:- count -= int(torch.count_nonzero(lengths > 0)) - 1- return count+ n_shift = token_count.shape[-1]+ n_rows = token_count.numel() // n_shift+ starts = torch.cumsum(lengths.long(), dim = 0)[:-1]+ rows = torch.div(starts, n_shift + 1, rounding_mode = "floor")+ cols = starts - rows * (n_shift + 1)+ keep = (cols > 0) & (rows < n_rows)+ token_count = token_count.reshape(n_rows, n_shift).clone()+ token_count[rows[keep], cols[keep] - 1] = False+ return int(token_count.sum())@pytest.mark.parametrize("wrapper", [enable_sample_packing, enable_padding_free_metadata])-def test_collator_keeps_boundary_targets_for_the_num_items_deduction(wrapper):+def test_collator_masks_boundary_targets_and_num_items_counts_them_once(wrapper):model = SimpleNamespace(max_seq_length = 16, children = lambda: [])trainer = SimpleNamespace(args = SimpleNamespace(remove_unused_columns = True),@@ -2337,25 +2342,68 @@ def test_collator_keeps_boundary_targets_for_the_num_items_deduction(wrapper):)wrapper(model, trainer)+ examples = [+ {"input_ids": [10, 11, 12], "seq_lengths": [2, 1]},+ {"input_ids": [13, 14, 15], "seq_lengths": [3]},+ ]+ batch = trainer.data_collator.torch_call(examples)+ # boundary labels match TRL >= 0.23.1+ assert batch["labels"].reshape(-1).tolist() == [-100, 11, -100, -100, 14, 15]+ assert batch["input_ids"].reshape(-1).tolist() == [10, 11, 12, 13, 14, 15]+ assert _zoo_num_items_in_batch(batch) == 3+ assert examples[0] == {"input_ids": [10, 11, 12], "seq_lengths": [2, 1]}++[email protected]("wrapper", [enable_sample_packing, enable_padding_free_metadata])+def test_collator_boundary_mask_is_a_no_op_on_trl_masked_labels(wrapper):+ class _MaskedPackingCollator(_UnmaskedPackingCollator):+ def torch_call(self, examples):+ batch = super().torch_call(examples)+ position_ids = torch.cat(+ [torch.arange(n) for ex in examples for n in ex["seq_lengths"]]+ ).view(1, -1)+ batch["labels"][position_ids == 0] = -100+ return batch++ model = SimpleNamespace(max_seq_length = 16, children = lambda: [])+ trainer = SimpleNamespace(+ args = SimpleNamespace(remove_unused_columns = True), data_collator = _MaskedPackingCollator()+ )+ wrapper(model, trainer)batch = trainer.data_collator.torch_call([{"input_ids": [10, 11, 12], "seq_lengths": [2, 1]},{"input_ids": [13, 14, 15], "seq_lengths": [3]},])- assert batch["labels"].reshape(-1).tolist() == [10, 11, 12, 13, 14, 15]- # docs [10,11] [12] [13,14,15] -> 1 + 0 + 2 real CE targets+ assert batch["labels"].reshape(-1).tolist() == [-100, 11, -100, -100, 14, 15]assert _zoo_num_items_in_batch(batch) == 3-# 4. idempotence, discriminating (an identity helper must not pass)+def test_collator_boundary_mask_leaves_unflattened_rows_alone():+ class _RowCollator(_UnmaskedPackingCollator):+ def torch_call(self, examples):+ rows = [ex["input_ids"] for ex in examples]+ return {"input_ids": torch.tensor(rows), "labels": torch.tensor(rows)}++ model = SimpleNamespace(max_seq_length = 16, children = lambda: [])+ trainer = SimpleNamespace(+ args = SimpleNamespace(remove_unused_columns = True), data_collator = _RowCollator()+ )+ enable_sample_packing(model, trainer)+ batch = trainer.data_collator.torch_call(+ [{"input_ids": [10, 11, 12]}, {"input_ids": [13, 14, 15]}]+ )+ assert batch["labels"].tolist() == [[10, 11, 12], [13, 14, 15]]++def test_guard_is_idempotent_and_actually_masks():lengths = torch.tensor([2, 1, 3], dtype = torch.int32)labels = torch.arange(6, dtype = torch.long).view(1, 6)once = mask_packed_boundary_labels(labels, lengths)twice = mask_packed_boundary_labels(once, lengths)assert torch.equal(twice, once)- # idempotence alone is trivial for an identity helper, so pin the values+ # an identity helper would also pass the idempotence checkassert once.reshape(-1).tolist() == [-100, 1, -100, -100, 4, 5]diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.pyindex afc585ae1f2..c8b4ecaabfe 100644--- a/unsloth/utils/packing.py+++ b/unsloth/utils/packing.py@@ -20,6 +20,7 @@ import loggingimport osimport sysfrom collections import OrderedDict+from collections.abc import Mappingfrom functools import wrapsfrom typing import Any, Iterable, Optional, Sequence, Tuple@@ -188,14 +189,13 @@ def configure_sample_packing(config):def configure_padding_free(config):- """Mutate an ``SFTConfig`` so TRL enables padding-free batching without packing."""_ensure_trl_warning_filter()setattr(config, "padding_free", True)setattr(config, "remove_unused_columns", False)def _enable_hf_packed_attention() -> None:- # Packed rows on the transformers modeling path run varlen instead of SDPA over a dense mask.+ # packed rows on the transformers modeling path run varlen instead of SDPA over a dense mask.try:from .hf_packed_attention import enable_hf_packed_attentionenable_hf_packed_attention()@@ -203,13 +203,20 @@ def _enable_hf_packed_attention() -> None:logging.getLogger(__name__).info(f"Unsloth: packed varlen attention not enabled ({exc})")+def _mask_padding_free_boundary_labels(batch) -> None:+ """TRL < 0.23.1 targets document starts; the zoo count excludes them; single row only."""+ labels = batch.get("labels") if isinstance(batch, Mapping) else None+ if not isinstance(labels, torch.Tensor) or labels.dim() != 2 or labels.shape[0] != 1:+ return+ batch["labels"] = mask_packed_boundary_labels(labels, batch["packed_seq_lengths"])++def enable_sample_packing(model,trainer,*,sequence_lengths_key: str = "seq_lengths",) -> None:- """Enable runtime support for packed batches on an existing trainer."""if model is None or trainer is None:raise ValueError("model and trainer must not be None")@@ -240,20 +247,14 @@ def enable_sample_packing(lengths = example.get(sequence_lengths_key)if isinstance(lengths, Iterable):seq_lengths.extend(int(length) for length in lengths)- # Fallback: infer lengths from tokenized inputs when metadata is absent.if not seq_lengths:for example in examples:ids = example.get("input_ids")if isinstance(ids, Iterable):seq_lengths.append(len(ids))if seq_lengths:- # Boundary labels are NOT masked here: unsloth_zoo's _unsloth_get_batch_samples counts- # num_items_in_batch off this batch and discounts the N-1 boundary targets itself, idempotently,- # zeroing those slots rather than subtracting a constant, so the count is unaffected by upstream- # masking (TRL >= 0.24's labels[position_ids == 0] = -100, completion-only masking,- # assistant_masks). Labels are left alone because the guard that needs these positions runs in the- # forward, off packed_seq_lengths.batch["packed_seq_lengths"] = torch.tensor(seq_lengths, dtype = torch.int32)+ _mask_padding_free_boundary_labels(batch)if "attention_mask" in batch:batch.pop("attention_mask")return batch@@ -263,7 +264,7 @@ def enable_sample_packing(def enable_padding_free_metadata(model, trainer):- """Inject seq-length metadata when padding-free batching is enabled without packing."""+ """supply sequence lengths for padding-free batches without packing."""collator = getattr(trainer, "data_collator", None)if (collator is None@@ -292,7 +293,7 @@ def enable_padding_free_metadata(model, trainer):if ids is None:continuelengths = [len(ids)]- # TRL's collator keys seq_lengths off examples[0] and reads every row: pass a copy, not the caller's row.+ # TRL reads all seq_lengths if examples[0] has them; copies preserve inputs.if collated is examples:collated = list(examples)collated[index] = {**example, "seq_lengths": lengths}@@ -300,21 +301,20 @@ def enable_padding_free_metadata(model, trainer):batch = original_torch_call(collated)if seq_lengths:- # Labels left alone for the same reason as enable_sample_packing: num_items_in_batch is counted off- # this batch and the zoo's discount of the boundary targets is idempotent.batch["packed_seq_lengths"] = torch.tensor(seq_lengths,dtype = torch.int32,)+ _mask_padding_free_boundary_labels(batch)return batchcollator.torch_call = torch_call_with_padding_free_metadatacollator._unsloth_padding_free_lengths_wrapped = True-# Experimental packing for hybrid linear-attention (Qwen3.5 / Qwen3-Next gated-delta, Nemotron-H Mamba2):-# a flattened batch leaks recurrent + conv state across sequences unless seq_idx / cu_seqlens reach the-# accelerated kernels, so this is env-gated and fails closed on the pure-torch fallbacks.+# experimental hybrid packing: Qwen3.5 / Qwen3-Next gated-delta and Nemotron-H Mamba2.+# recurrent and conv state leak unless seq_idx / cu_seqlens reach accelerated kernels.+# env-gated; pure-torch fallbacks fail closed to prevent cross-sequence state leaks._MAMBA2_FUSED_NAMES = ("mamba2_split_conv1d_scan_combined","mamba_split_conv1d_scan_combined",