Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Mask packed document boundary labels for TRL older than 0.23.1 (#13324)

Daniel Han7aa07618b964
diff --git a/tests/utils/test_packing.py b/tests/utils/test_packing.py
index 807c00ff0e9..26c30a0bcc3 100644
--- a/tests/utils/test_packing.py
+++ b/tests/utils/test_packing.py
@@ -2301,11 +2301,9 @@ def test_fused_ce_branch_masks_packed_boundaries(monkeypatch, module_name):
assert labels.reshape(-1).tolist() == list(range(seq))
-# 3. the collator wrappers must leave boundary targets in place: unsloth_zoo counts num_items_in_batch off this
-# batch and already deducts them
+# wrappers match TRL >= 0.23.1 masking; the zoo count includes each real target once
class _UnmaskedPackingCollator:
- """Padding-free collator that does NOT pre-mask boundaries, like TRL < 0.24 - a test
- built on TRL 0.24+ output would pass either way."""
+ """padding-free collator with unmasked boundaries, like TRL < 0.23.1."""
def __init__(self):
self.padding_free = True
@@ -2320,16 +2318,23 @@ class _UnmaskedPackingCollator:
def _zoo_num_items_in_batch(batch):
- """The count unsloth_zoo._unsloth_get_batch_samples derives from a batch."""
- count = int((batch["labels"][..., 1:] != -100).sum())
+ """unsloth_zoo._unsloth_get_batch_samples' count: boundary slots zeroed, not subtracted."""
+ token_count = batch["labels"][..., 1:] != -100
lengths = batch.get("packed_seq_lengths")
if lengths is not None:
- count -= int(torch.count_nonzero(lengths > 0)) - 1
- return count
+ n_shift = token_count.shape[-1]
+ n_rows = token_count.numel() // n_shift
+ starts = torch.cumsum(lengths.long(), dim = 0)[:-1]
+ rows = torch.div(starts, n_shift + 1, rounding_mode = "floor")
+ cols = starts - rows * (n_shift + 1)
+ keep = (cols > 0) & (rows < n_rows)
+ token_count = token_count.reshape(n_rows, n_shift).clone()
+ token_count[rows[keep], cols[keep] - 1] = False
+ return int(token_count.sum())
@pytest.mark.parametrize("wrapper", [enable_sample_packing, enable_padding_free_metadata])
-def test_collator_keeps_boundary_targets_for_the_num_items_deduction(wrapper):
+def test_collator_masks_boundary_targets_and_num_items_counts_them_once(wrapper):
model = SimpleNamespace(max_seq_length = 16, children = lambda: [])
trainer = SimpleNamespace(
args = SimpleNamespace(remove_unused_columns = True),
@@ -2337,25 +2342,68 @@ def test_collator_keeps_boundary_targets_for_the_num_items_deduction(wrapper):
)
wrapper(model, trainer)
+ examples = [
+ {"input_ids": [10, 11, 12], "seq_lengths": [2, 1]},
+ {"input_ids": [13, 14, 15], "seq_lengths": [3]},
+ ]
+ batch = trainer.data_collator.torch_call(examples)
+ # boundary labels match TRL >= 0.23.1
+ assert batch["labels"].reshape(-1).tolist() == [-100, 11, -100, -100, 14, 15]
+ assert batch["input_ids"].reshape(-1).tolist() == [10, 11, 12, 13, 14, 15]
+ assert _zoo_num_items_in_batch(batch) == 3
+ assert examples[0] == {"input_ids": [10, 11, 12], "seq_lengths": [2, 1]}
+
+
[email protected]("wrapper", [enable_sample_packing, enable_padding_free_metadata])
+def test_collator_boundary_mask_is_a_no_op_on_trl_masked_labels(wrapper):
+ class _MaskedPackingCollator(_UnmaskedPackingCollator):
+ def torch_call(self, examples):
+ batch = super().torch_call(examples)
+ position_ids = torch.cat(
+ [torch.arange(n) for ex in examples for n in ex["seq_lengths"]]
+ ).view(1, -1)
+ batch["labels"][position_ids == 0] = -100
+ return batch
+
+ model = SimpleNamespace(max_seq_length = 16, children = lambda: [])
+ trainer = SimpleNamespace(
+ args = SimpleNamespace(remove_unused_columns = True), data_collator = _MaskedPackingCollator()
+ )
+ wrapper(model, trainer)
batch = trainer.data_collator.torch_call(
[
{"input_ids": [10, 11, 12], "seq_lengths": [2, 1]},
{"input_ids": [13, 14, 15], "seq_lengths": [3]},
]
)
- assert batch["labels"].reshape(-1).tolist() == [10, 11, 12, 13, 14, 15]
- # docs [10,11] [12] [13,14,15] -> 1 + 0 + 2 real CE targets
+ assert batch["labels"].reshape(-1).tolist() == [-100, 11, -100, -100, 14, 15]
assert _zoo_num_items_in_batch(batch) == 3
-# 4. idempotence, discriminating (an identity helper must not pass)
+def test_collator_boundary_mask_leaves_unflattened_rows_alone():
+ class _RowCollator(_UnmaskedPackingCollator):
+ def torch_call(self, examples):
+ rows = [ex["input_ids"] for ex in examples]
+ return {"input_ids": torch.tensor(rows), "labels": torch.tensor(rows)}
+
+ model = SimpleNamespace(max_seq_length = 16, children = lambda: [])
+ trainer = SimpleNamespace(
+ args = SimpleNamespace(remove_unused_columns = True), data_collator = _RowCollator()
+ )
+ enable_sample_packing(model, trainer)
+ batch = trainer.data_collator.torch_call(
+ [{"input_ids": [10, 11, 12]}, {"input_ids": [13, 14, 15]}]
+ )
+ assert batch["labels"].tolist() == [[10, 11, 12], [13, 14, 15]]
+
+
def test_guard_is_idempotent_and_actually_masks():
lengths = torch.tensor([2, 1, 3], dtype = torch.int32)
labels = torch.arange(6, dtype = torch.long).view(1, 6)
once = mask_packed_boundary_labels(labels, lengths)
twice = mask_packed_boundary_labels(once, lengths)
assert torch.equal(twice, once)
- # idempotence alone is trivial for an identity helper, so pin the values
+ # an identity helper would also pass the idempotence check
assert once.reshape(-1).tolist() == [-100, 1, -100, -100, 4, 5]
diff --git a/unsloth/utils/packing.py b/unsloth/utils/packing.py
index afc585ae1f2..c8b4ecaabfe 100644
--- a/unsloth/utils/packing.py
+++ b/unsloth/utils/packing.py
@@ -20,6 +20,7 @@ import logging
import os
import sys
from collections import OrderedDict
+from collections.abc import Mapping
from functools import wraps
from typing import Any, Iterable, Optional, Sequence, Tuple
@@ -188,14 +189,13 @@ def configure_sample_packing(config):
def configure_padding_free(config):
- """Mutate an ``SFTConfig`` so TRL enables padding-free batching without packing."""
_ensure_trl_warning_filter()
setattr(config, "padding_free", True)
setattr(config, "remove_unused_columns", False)
def _enable_hf_packed_attention() -> None:
- # Packed rows on the transformers modeling path run varlen instead of SDPA over a dense mask.
+ # packed rows on the transformers modeling path run varlen instead of SDPA over a dense mask.
try:
from .hf_packed_attention import enable_hf_packed_attention
enable_hf_packed_attention()
@@ -203,13 +203,20 @@ def _enable_hf_packed_attention() -> None:
logging.getLogger(__name__).info(f"Unsloth: packed varlen attention not enabled ({exc})")
+def _mask_padding_free_boundary_labels(batch) -> None:
+ """TRL < 0.23.1 targets document starts; the zoo count excludes them; single row only."""
+ labels = batch.get("labels") if isinstance(batch, Mapping) else None
+ if not isinstance(labels, torch.Tensor) or labels.dim() != 2 or labels.shape[0] != 1:
+ return
+ batch["labels"] = mask_packed_boundary_labels(labels, batch["packed_seq_lengths"])
+
+
def enable_sample_packing(
model,
trainer,
*,
sequence_lengths_key: str = "seq_lengths",
) -> None:
- """Enable runtime support for packed batches on an existing trainer."""
if model is None or trainer is None:
raise ValueError("model and trainer must not be None")
@@ -240,20 +247,14 @@ def enable_sample_packing(
lengths = example.get(sequence_lengths_key)
if isinstance(lengths, Iterable):
seq_lengths.extend(int(length) for length in lengths)
- # Fallback: infer lengths from tokenized inputs when metadata is absent.
if not seq_lengths:
for example in examples:
ids = example.get("input_ids")
if isinstance(ids, Iterable):
seq_lengths.append(len(ids))
if seq_lengths:
- # Boundary labels are NOT masked here: unsloth_zoo's _unsloth_get_batch_samples counts
- # num_items_in_batch off this batch and discounts the N-1 boundary targets itself, idempotently,
- # zeroing those slots rather than subtracting a constant, so the count is unaffected by upstream
- # masking (TRL >= 0.24's labels[position_ids == 0] = -100, completion-only masking,
- # assistant_masks). Labels are left alone because the guard that needs these positions runs in the
- # forward, off packed_seq_lengths.
batch["packed_seq_lengths"] = torch.tensor(seq_lengths, dtype = torch.int32)
+ _mask_padding_free_boundary_labels(batch)
if "attention_mask" in batch:
batch.pop("attention_mask")
return batch
@@ -263,7 +264,7 @@ def enable_sample_packing(
def enable_padding_free_metadata(model, trainer):
- """Inject seq-length metadata when padding-free batching is enabled without packing."""
+ """supply sequence lengths for padding-free batches without packing."""
collator = getattr(trainer, "data_collator", None)
if (
collator is None
@@ -292,7 +293,7 @@ def enable_padding_free_metadata(model, trainer):
if ids is None:
continue
lengths = [len(ids)]
- # TRL's collator keys seq_lengths off examples[0] and reads every row: pass a copy, not the caller's row.
+ # TRL reads all seq_lengths if examples[0] has them; copies preserve inputs.
if collated is examples:
collated = list(examples)
collated[index] = {**example, "seq_lengths": lengths}
@@ -300,21 +301,20 @@ def enable_padding_free_metadata(model, trainer):
batch = original_torch_call(collated)
if seq_lengths:
- # Labels left alone for the same reason as enable_sample_packing: num_items_in_batch is counted off
- # this batch and the zoo's discount of the boundary targets is idempotent.
batch["packed_seq_lengths"] = torch.tensor(
seq_lengths,
dtype = torch.int32,
)
+ _mask_padding_free_boundary_labels(batch)
return batch
collator.torch_call = torch_call_with_padding_free_metadata
collator._unsloth_padding_free_lengths_wrapped = True
-# Experimental packing for hybrid linear-attention (Qwen3.5 / Qwen3-Next gated-delta, Nemotron-H Mamba2):
-# a flattened batch leaks recurrent + conv state across sequences unless seq_idx / cu_seqlens reach the
-# accelerated kernels, so this is env-gated and fails closed on the pure-torch fallbacks.
+# experimental hybrid packing: Qwen3.5 / Qwen3-Next gated-delta and Nemotron-H Mamba2.
+# recurrent and conv state leak unless seq_idx / cu_seqlens reach accelerated kernels.
+# env-gated; pure-torch fallbacks fail closed to prevent cross-sequence state leaks.
_MAMBA2_FUSED_NAMES = (
"mamba2_split_conv1d_scan_combined",
"mamba_split_conv1d_scan_combined",