Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Opt-in token-balanced and token-budget SFT micro-batches (#13878)

Daniel Haneac949c8a0ef
diff --git a/tests/test_token_budget_batches.py b/tests/test_token_budget_batches.py
new file mode 100644
index 00000000000..c7af5080be3
--- /dev/null
+++ b/tests/test_token_budget_batches.py
@@ -0,0 +1,728 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_TOKEN_BALANCE / UNSLOTH_TOKEN_BUDGET: partitioner, step invariants, exact split loss. CPU only."""
+
+from __future__ import annotations
+
+import contextlib
+import importlib.util
+import itertools
+import random
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "utils" / "token_budget.py"
+_spec = importlib.util.spec_from_file_location("_unsloth_token_budget", _PATH)
+tb = importlib.util.module_from_spec(_spec)
+sys.modules[_spec.name] = tb
+_spec.loader.exec_module(tb)
+
+
+def _sums(values, groups):
+ return [sum(values[i] for i in g) for g in groups]
+
+
+def _best_spread(values, k):
+ best = None
+ for labels in itertools.product(range(k), repeat = len(values)):
+ sums = [0] * k
+ for v, g in zip(values, labels):
+ sums[g] += v
+ spread = max(sums) - min(sums)
+ best = spread if best is None else min(best, spread)
+ return best
+
+
[email protected]("seed", range(20))
+def test_karmarkar_karp_is_a_partition_close_to_optimal(seed):
+ rng = random.Random(seed)
+ values = [rng.randint(1, 100) for _ in range(rng.randint(3, 9))]
+ k = rng.choice([2, 3])
+ groups = tb.karmarkar_karp(values, k)
+ assert sorted(i for g in groups for i in g) == list(range(len(values)))
+ sums = _sums(values, groups)
+ assert max(sums) - min(sums) <= _best_spread(values, k) + max(values)
+
+
+def test_karmarkar_karp_equal_size_and_classic_example():
+ groups = tb.karmarkar_karp([8, 7, 6, 5, 4], 2)
+ assert sorted(_sums([8, 7, 6, 5, 4], groups)) == [14, 16]
+ rng = random.Random(0)
+ values = [rng.randint(1, 4096) for _ in range(64)]
+ groups = tb.karmarkar_karp(values, 8, equal_size = True)
+ assert all(len(g) == 8 for g in groups)
+ assert sorted(i for g in groups for i in g) == list(range(64))
+ sums = _sums(values, groups)
+ assert max(sums) - min(sums) < 0.05 * (sum(values) / 8)
+ with pytest.raises(ValueError):
+ tb.karmarkar_karp(values[:63], 8, equal_size = True)
+
+
+def _layout(order, bs, ga, ws):
+ """accelerate BatchSamplerShard: global batch b -> rank b % ws, the rank's (b // ws)-th batch."""
+ batches = [order[i : i + bs] for i in range(0, len(order), bs)]
+ return [[batches[g * ws + r] for g in range(ga)] for r in range(ws)]
+
+
+def _num_items(labels, batches):
+ # HF / zoo count: shifted labels that are not -100, summed over the step's micro-batches.
+ return sum(int((labels[i][1:] != -100).sum()) for batch in batches for i in batch)
+
+
[email protected]("ws,ga,bs", [(1, 4, 2), (2, 2, 3), (4, 2, 2), (2, 1, 4)])
[email protected]("cross_rank", [True, False])
+def test_step_rows_and_num_items_are_unchanged(ws, ga, bs, cross_rank):
+ rng = random.Random(ws * 100 + ga * 10 + bs)
+ n = ws * ga * bs
+ lengths = [rng.choice([16, 32, 512, 2048]) for _ in range(n)]
+ labels = []
+ for length in lengths:
+ row = torch.randint(0, 50, (length,))
+ row[: rng.randint(0, length - 1)] = -100
+ labels.append(row)
+ order = list(range(n))
+ rng.shuffle(order)
+ balanced = tb.balance_step_indices(order, lengths, bs, ga, ws, cross_rank)
+ assert sorted(balanced) == sorted(order)
+
+ before = _layout(order, bs, ga, ws)
+ after = _layout(balanced, bs, ga, ws)
+ assert sum(_num_items(labels, r) for r in before) == sum(_num_items(labels, r) for r in after)
+ assert all(len(b) == bs for r in after for b in r)
+ if not cross_rank or ws == 1:
+ for r in range(ws):
+ assert sorted(sum(before[r], [])) == sorted(sum(after[r], []))
+ assert _num_items(labels, before[r]) == _num_items(labels, after[r])
+
+
+def test_balancing_evens_out_rank_and_slot_tokens():
+ bs, ga, ws = 4, 4, 4
+ lengths = [4096] * 8 + [64] * (bs * ga * ws - 8)
+ order = list(range(len(lengths)))
+ before = _layout(order, bs, ga, ws)
+ after = _layout(tb.balance_step_indices(order, lengths, bs, ga, ws, True), bs, ga, ws)
+
+ def rank_tokens(layout):
+ return [sum(lengths[i] for b in r for i in b) for r in layout]
+
+ def slot_max(layout):
+ return max(sum(lengths[i] for i in b) for r in layout for b in r)
+
+ assert max(rank_tokens(before)) - min(rank_tokens(before)) > 10_000
+ assert max(rank_tokens(after)) - min(rank_tokens(after)) == 0
+ assert slot_max(after) < slot_max(before)
+
+
+def test_sampler_keeps_the_index_multiset_and_forwards_set_epoch():
+ lengths = [random.Random(i).randint(1, 999) for i in range(23)]
+ base = torch.utils.data.RandomSampler(range(23), generator = torch.Generator().manual_seed(0))
+ sampler = tb.TokenBalancedSampler(base, lengths, batch_size = 2, grad_accum = 2, world_size = 2)
+ out = list(sampler)
+ assert len(sampler) == 23 and sorted(out) == list(range(23))
+ calls = []
+ base.set_epoch = calls.append
+ sampler.set_epoch(3)
+ assert calls == [3]
+ assert sampler.generator is base.generator
+
+
+def _padding_free_batch(segments, mask_boundaries = True):
+ total = sum(segments)
+ torch.manual_seed(0)
+ input_ids = torch.randint(1, 30, (1, total))
+ labels = input_ids.clone()
+ position_ids = torch.cat([torch.arange(s) for s in segments]).unsqueeze(0)
+ if mask_boundaries:
+ labels[position_ids == 0] = -100
+ return {
+ "input_ids": input_ids,
+ "labels": labels,
+ "position_ids": position_ids,
+ "packed_seq_lengths": torch.tensor(segments, dtype = torch.int32),
+ }
+
+
+class _TinyLM(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ self.emb = torch.nn.Embedding(32, 8)
+ self.head = torch.nn.Linear(8, 32)
+
+ def loss(self, batch, num_items):
+ logits = self.head(torch.tanh(self.emb(batch["input_ids"])))
+ shift_logits = logits[:, :-1].reshape(-1, 32)
+ shift_labels = batch["labels"][:, 1:].reshape(-1)
+ return (
+ torch.nn.functional.cross_entropy(
+ shift_logits, shift_labels, ignore_index = -100, reduction = "sum"
+ )
+ / num_items
+ )
+
+
+def _grads(model, batches, num_items):
+ model.zero_grad()
+ total = 0.0
+ for batch in batches:
+ loss = model.loss(batch, num_items)
+ loss.backward()
+ total += float(loss)
+ return total, [p.grad.clone() for p in model.parameters()]
+
+
+def test_padding_free_split_is_exact_and_respects_the_budget():
+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4])
+ chunks = tb.split_batch_by_token_budget(batch, 16)
+ assert len(chunks) > 1
+ for chunk in chunks:
+ assert chunk["input_ids"].shape[1] <= 16 or len(chunk["packed_seq_lengths"]) == 1
+ assert int(chunk["packed_seq_lengths"].sum()) == chunk["input_ids"].shape[1]
+ assert int(chunk["position_ids"][0, 0]) == 0
+ for key in ("input_ids", "labels", "position_ids"):
+ assert torch.equal(torch.cat([c[key] for c in chunks], dim = 1), batch[key])
+
+ num_items = int((batch["labels"][:, 1:] != -100).sum())
+ model = _TinyLM()
+ full_loss, full_grads = _grads(model, [batch], num_items)
+ split_loss, split_grads = _grads(model, chunks, num_items)
+ assert abs(full_loss - split_loss) < 1e-5
+ for a, b in zip(full_grads, split_grads):
+ torch.testing.assert_close(a, b, atol = 1e-6, rtol = 1e-5)
+
+
+def test_split_refuses_when_a_cut_would_drop_a_real_target():
+ batch = _padding_free_batch([5, 9, 3, 12], mask_boundaries = False)
+ assert tb.split_batch_by_token_budget(batch, 10) == [batch]
+
+
+def test_split_leaves_small_unknown_and_vision_batches_alone():
+ batch = _padding_free_batch([5, 6])
+ assert tb.split_batch_by_token_budget(batch, 64) == [batch]
+ vision = {"input_ids": torch.ones(4, 8, dtype = torch.long), "pixel_values": torch.ones(7, 3)}
+ assert tb.split_batch_by_token_budget(vision, 8) == [vision]
+
+
+def test_padded_rows_split_is_exact():
+ torch.manual_seed(1)
+ input_ids = torch.randint(1, 30, (6, 10))
+ labels = input_ids.clone()
+ labels[:, :3] = -100
+ batch = {"input_ids": input_ids, "labels": labels, "attention_mask": torch.ones_like(input_ids)}
+ chunks = tb.split_batch_by_token_budget(batch, 20)
+ assert [c["input_ids"].shape[0] for c in chunks] == [2, 2, 2]
+ num_items = int((labels[:, 1:] != -100).sum())
+ model = _TinyLM()
+ full_loss, full_grads = _grads(model, [batch], num_items)
+ split_loss, split_grads = _grads(model, chunks, num_items)
+ assert abs(full_loss - split_loss) < 1e-5
+ for a, b in zip(full_grads, split_grads):
+ torch.testing.assert_close(a, b, atol = 1e-6, rtol = 1e-5)
+
+
+class _FakeAccelerator:
+ distributed_type = "DistributedType.MULTI_GPU"
+
+ def __init__(self):
+ self.no_sync_calls = 0
+
+ @contextlib.contextmanager
+ def no_sync(self, model):
+ self.no_sync_calls += 1
+ yield
+
+
+def _fake_trainer():
+ seen = []
+
+ def training_step(
+ model,
+ inputs,
+ num_items_in_batch = None,
+ ):
+ seen.append((inputs["input_ids"].shape[1], num_items_in_batch))
+ return torch.tensor(float(inputs["input_ids"].shape[1]) / num_items_in_batch)
+
+ trainer = SimpleNamespace(
+ training_step = training_step,
+ get_train_dataloader = lambda: None,
+ accelerator = _FakeAccelerator(),
+ args = SimpleNamespace(gradient_accumulation_steps = 1),
+ model_accepts_loss_kwargs = True,
+ label_smoother = None,
+ )
+ return trainer, seen
+
+
+def test_install_is_a_no_op_without_the_env(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)
+ monkeypatch.delenv("UNSLOTH_TOKEN_BALANCE", raising = False)
+ trainer, _ = _fake_trainer()
+ step, loader = trainer.training_step, trainer.get_train_dataloader
+ tb.install_token_budget(trainer)
+ assert trainer.training_step is step and trainer.get_train_dataloader is loader
+
+
+def test_budgeted_training_step_sums_chunks_and_syncs_once(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_TOKEN_BUDGET", "16")
+ monkeypatch.delenv("UNSLOTH_TOKEN_BALANCE", raising = False)
+ trainer, seen = _fake_trainer()
+ tb.install_token_budget(trainer)
+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4])
+ loss = trainer.training_step(None, batch, 100)
+ assert len(seen) > 1 and all(n == 100 for _, n in seen)
+ assert sum(width for width, _ in seen) == 40
+ assert abs(float(loss) - 0.40) < 1e-6
+ assert trainer.accelerator.no_sync_calls == len(seen) - 1
+ assert trainer._unsloth_token_budget_splits == 1
+ # Without a token count the loss is a per-call mean, so the batch is not split.
+ seen.clear()
+ with pytest.raises(TypeError):
+ # The fake divides by num_items_in_batch: reaching it unsplit with None proves the pass-through.
+ trainer.training_step(None, batch, None)
+ assert seen == [(40, None)]
+
+
+def test_balanced_dataloader_end_to_end_with_accelerate_shards(monkeypatch):
+ accelerate_data_loader = pytest.importorskip("accelerate.data_loader")
+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")
+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)
+ datasets = pytest.importorskip("datasets")
+ rng = random.Random(0)
+ lengths = [rng.choice([8, 8, 8, 600]) for _ in range(64)]
+ dataset = datasets.Dataset.from_dict({"input_ids": [[1] * n for n in lengths]})
+ bs, ga, ws = 2, 4, 2
+
+ def make(rank):
+ sampler = torch.utils.data.SequentialSampler(dataset)
+ inner = torch.utils.data.BatchSampler(sampler, bs, drop_last = False)
+ shard = accelerate_data_loader.BatchSamplerShard(
+ inner, num_processes = ws, process_index = rank, split_batches = False
+ )
+ loader = SimpleNamespace(batch_sampler = shard, dataset = dataset)
+ trainer = SimpleNamespace(
+ training_step = None,
+ get_train_dataloader = lambda: loader,
+ args = SimpleNamespace(
+ gradient_accumulation_steps = ga, average_tokens_across_devices = True
+ ),
+ model_accepts_loss_kwargs = True,
+ )
+ tb.install_token_budget(trainer)
+ trainer.get_train_dataloader()
+ assert trainer._unsloth_token_balance_active
+ return list(shard)
+
+ per_rank = [make(r) for r in range(ws)]
+ steps = len(per_rank[0]) // ga
+ for s in range(steps):
+ step_rows = [i for r in range(ws) for b in per_rank[r][s * ga : (s + 1) * ga] for i in b]
+ assert sorted(step_rows) == list(range(s * ga * ws * bs, (s + 1) * ga * ws * bs))
+ tokens = [
+ sum(lengths[i] for b in per_rank[r][s * ga : (s + 1) * ga] for i in b)
+ for r in range(ws)
+ ]
+ assert max(tokens) - min(tokens) <= max(lengths)
+
+
+class _MeanLabelSmoother:
+ # transformers < 5 LabelSmoother.__call__ signature: no num_items_in_batch, returns a per-call mean.
+ def __call__(
+ self,
+ model_output,
+ labels,
+ shift_labels = False,
+ ):
+ raise AssertionError("not called")
+
+
+class _CountLabelSmoother:
+ def __call__(
+ self,
+ model_output,
+ labels,
+ shift_labels = False,
+ num_items_in_batch = None,
+ ):
+ raise AssertionError("not called")
+
+
+def _moe_model(coef):
+ config = SimpleNamespace(output_router_logits = True, router_aux_loss_coef = coef)
+ return SimpleNamespace(config = config)
+
+
+ "override,splits",
+ [
+ ({}, True),
+ ({"label_smoother": _CountLabelSmoother()}, True),
+ ({"label_smoother": _MeanLabelSmoother()}, False),
+ ({"model_accepts_loss_kwargs": False}, False),
+ ({"aux_loss_enabled": True}, False),
+ ({"model": _moe_model(0.01)}, False),
+ ({"model": _moe_model(0.0)}, True),
+ ],
+)
+def test_split_steps_aside_when_chunk_losses_do_not_sum_exactly(
+ monkeypatch, capsys, override, splits
+):
+ monkeypatch.setenv("UNSLOTH_TOKEN_BUDGET", "16")
+ monkeypatch.delenv("UNSLOTH_TOKEN_BALANCE", raising = False)
+ trainer, seen = _fake_trainer()
+ for key, value in override.items():
+ setattr(trainer, key, value)
+ tb.install_token_budget(trainer)
+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4])
+ trainer.training_step(None, batch, 100)
+ trainer.training_step(None, batch, 100)
+ assert (len(seen) > 2) == splits
+ if not splits:
+ assert seen == [(40, 100), (40, 100)]
+ assert capsys.readouterr().out.count("UNSLOTH_TOKEN_BUDGET not splitting") == 1
+
+
+def test_split_refuses_media_keys_and_per_sample_metadata():
+ rows = {
+ "input_ids": torch.ones(4, 8, dtype = torch.long),
+ "labels": torch.ones(4, 8, dtype = torch.long),
+ }
+ assert len(tb.split_batch_by_token_budget(rows, 8)) == 4
+ # Flattened images that happen to total B rows still must not be paired row by row.
+ for extra in (
+ {"pixel_values": torch.ones(4, 3)},
+ {"image_grid_thw": torch.ones(4, 3, dtype = torch.long)},
+ {"input_values": torch.ones(4, 16000)},
+ {"num_images": [0, 2, 1, 1]},
+ ):
+ batch = {**rows, **extra}
+ assert tb.split_batch_by_token_budget(batch, 8) == [batch]
+ packed = {**_padding_free_batch([5, 9, 3, 12]), "pixel_values": torch.ones(1, 29, 3)}
+ assert tb.split_batch_by_token_budget(packed, 10) == [packed]
+
+
+def _shard_steps(n, bs, ga, ws, balance, **trainer_kwargs):
+ accelerate_data_loader = pytest.importorskip("accelerate.data_loader")
+ datasets = pytest.importorskip("datasets")
+ rng = random.Random(1)
+ lengths = [rng.choice([8, 8, 8, 600]) for _ in range(n)]
+ dataset = datasets.Dataset.from_dict({"input_ids": [[1] * k for k in lengths]})
+ per_rank, active = [], []
+ for rank in range(ws):
+ base = torch.utils.data.RandomSampler(dataset, generator = torch.Generator().manual_seed(3))
+ inner = torch.utils.data.BatchSampler(base, bs, drop_last = False)
+ shard = accelerate_data_loader.BatchSamplerShard(
+ inner, num_processes = ws, process_index = rank, split_batches = False, even_batches = True
+ )
+ if balance:
+ loader = SimpleNamespace(batch_sampler = shard, dataset = dataset)
+ kwargs = {"model_accepts_loss_kwargs": True, **trainer_kwargs}
+ optim = kwargs.pop("args_optim", "adamw_torch")
+ trainer = SimpleNamespace(
+ training_step = None,
+ get_train_dataloader = lambda: loader,
+ args = SimpleNamespace(
+ gradient_accumulation_steps = ga, average_tokens_across_devices = True, optim = optim
+ ),
+ **kwargs,
+ )
+ tb.install_token_budget(trainer)
+ trainer.get_train_dataloader()
+ active.append(trainer._unsloth_token_balance_active)
+ per_rank.append(list(shard))
+ steps = []
+ for s in range(-(-len(per_rank[0]) // ga)):
+ steps.append(
+ sorted(i for r in range(ws) for b in per_rank[r][s * ga : (s + 1) * ga] for i in b)
+ )
+ return steps, active
+
+
[email protected]("n", [61, 64, 67])
+def test_balance_keeps_every_step_including_the_even_batches_tail(monkeypatch, n):
+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")
+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)
+ bs, ga, ws = 2, 4, 2
+ plain, _ = _shard_steps(n, bs, ga, ws, balance = False)
+ balanced, active = _shard_steps(n, bs, ga, ws, balance = True)
+ assert all(active)
+ assert balanced == plain
+
+
+ "override",
+ [
+ {"model_accepts_loss_kwargs": False},
+ {"label_smoother": _MeanLabelSmoother()},
+ {"aux_loss_enabled": True},
+ ],
+)
+def test_balance_steps_aside_when_slot_grouping_changes_the_loss(monkeypatch, capsys, override):
+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")
+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)
+ plain, _ = _shard_steps(64, 2, 4, 1, balance = False)
+ balanced, active = _shard_steps(64, 2, 4, 1, balance = True, **override)
+ assert active == [False] and balanced == plain
+ assert "UNSLOTH_TOKEN_BALANCE ignored" in capsys.readouterr().out
+
+
+def test_real_label_smoother_is_classified_by_its_signature():
+ trainer_pt_utils = pytest.importorskip("transformers.trainer_pt_utils")
+ import transformers
+
+ smoother = trainer_pt_utils.LabelSmoother(epsilon = 0.1)
+ trainer = SimpleNamespace(model_accepts_loss_kwargs = True, label_smoother = smoother)
+ uses_count = tb._loss_uses_step_count(trainer)
+ if int(transformers.__version__.split(".")[0]) < 5:
+ assert not uses_count
+ assert uses_count == (
+ "num_items_in_batch" in __import__("inspect").signature(smoother.__call__).parameters
+ )
+
+
+def test_split_refuses_pairwise_masks():
+ batch = _padding_free_batch([5, 9, 3, 12])
+ total = batch["input_ids"].shape[1]
+ batch["attention_mask"] = torch.ones(1, total, total, dtype = torch.bool).tril()
+ assert tb.split_batch_by_token_budget(batch, 10) == [batch]
+
+
+class _CountingTensor(torch.Tensor):
+ host_reads = 0
+
+ @classmethod
+ def __torch_function__(
+ cls,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ if func in (
+ torch.Tensor.__int__,
+ torch.Tensor.item,
+ torch.Tensor.tolist,
+ torch.Tensor.__bool__,
+ ):
+ cls.host_reads += 1
+ return super().__torch_function__(func, types, args, kwargs or {})
+
+
+def test_split_reads_boundary_labels_in_one_transfer():
+ batch = _padding_free_batch([3] * 40)
+ batch["labels"] = batch["labels"].as_subclass(_CountingTensor)
+ _CountingTensor.host_reads = 0
+ chunks = tb.split_batch_by_token_budget(batch, 6)
+ assert len(chunks) == 20
+ assert _CountingTensor.host_reads <= 1
+
+
+def test_lomo_steps_aside_for_split_and_balance(monkeypatch, capsys):
+ monkeypatch.setenv("UNSLOTH_TOKEN_BUDGET", "16")
+ monkeypatch.delenv("UNSLOTH_TOKEN_BALANCE", raising = False)
+ for optim, neftune in (
+ ("lomo", None),
+ ("adalomo", None),
+ ("galore_adamw_layerwise", None),
+ ("adamw_torch", 5.0),
+ ):
+ trainer, seen = _fake_trainer()
+ trainer.args.optim = optim
+ trainer.args.neftune_noise_alpha = neftune
+ tb.install_token_budget(trainer)
+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4])
+ trainer.training_step(None, batch, 100)
+ trainer.training_step(None, batch, 100)
+ assert seen == [(40, 100), (40, 100)]
+ assert capsys.readouterr().out.count("UNSLOTH_TOKEN_BUDGET not splitting") == 1
+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")
+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)
+ plain, _ = _shard_steps(64, 2, 4, 1, balance = False)
+ balanced, active = _shard_steps(64, 2, 4, 1, balance = True, args_optim = "lomo")
+ assert active == [False] and balanced == plain
+
+
+def test_balance_steps_aside_for_multi_process_dispatched_dataloaders(monkeypatch, capsys):
+ accelerate_data_loader = pytest.importorskip("accelerate.data_loader")
+ datasets = pytest.importorskip("datasets")
+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")
+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)
+ dataset = datasets.Dataset.from_dict({"input_ids": [[1] * (1 + i % 7) for i in range(32)]})
+ loader = accelerate_data_loader.DataLoaderDispatcher(dataset, batch_size = 2)
+ loader.state = SimpleNamespace(num_processes = 2)
+ trainer = SimpleNamespace(
+ training_step = None,
+ get_train_dataloader = lambda: loader,
+ args = SimpleNamespace(gradient_accumulation_steps = 4, average_tokens_across_devices = True),
+ model_accepts_loss_kwargs = True,
+ )
+ tb.install_token_budget(trainer)
+ trainer.get_train_dataloader()
+ assert trainer._unsloth_token_balance_active is False
+ assert not isinstance(loader.batch_sampler.sampler, tb.TokenBalancedSampler)
+ assert "UNSLOTH_TOKEN_BALANCE ignored" in capsys.readouterr().out
+
+
+def test_split_checks_boundary_labels_even_with_shift_labels():
+ # Unsloth's fast causal LM path builds its loss from labels, not shift_labels.
+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4], mask_boundaries = False)
+ batch["shift_labels"] = torch.roll(batch["labels"], -1, dims = 1)
+ assert len(tb.split_batch_by_token_budget(batch, 16)) == 1
+
+
+def test_dataset_lengths_are_read_in_batches():
+ datasets = pytest.importorskip("datasets")
+ dataset = datasets.Dataset.from_dict(
+ {"input_ids": [[1] * (1 + i % 7) for i in range(10_000)], "x": [0] * 10_000}
+ )
+ dataset = dataset.shuffle(seed = 0)
+ assert tb._dataset_lengths(dataset) == [len(ids) for ids in dataset["input_ids"]]
+
+
+_DDP_WORKER = r"""
+import contextlib, importlib.util, os, sys
+from types import SimpleNamespace
+import torch, torch.distributed as dist
+
+spec = importlib.util.spec_from_file_location("_tb", sys.argv[1])
+tb = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(tb)
+dist.init_process_group("gloo")
+rank = dist.get_rank()
+segments = [[5, 9, 3, 12, 7, 4], [int(x) for x in sys.argv[2].split(",")]][rank]
+total = sum(segments)
+input_ids = torch.randint(1, 30, (1, total))
+position_ids = torch.cat([torch.arange(s) for s in segments]).unsqueeze(0)
+labels = input_ids.clone()
+labels[position_ids == 0] = -100
+batch = dict(input_ids = input_ids, labels = labels, position_ids = position_ids,
+ packed_seq_lengths = torch.tensor(segments, dtype = torch.int32))
+calls = []
+
+def training_step(model, inputs, num_items_in_batch = None):
+ calls.append(inputs["input_ids"].shape[1])
+ # Stands in for TRL's per-compute_loss metric gather: one collective per call.
+ dist.all_reduce(torch.ones(1))
+ return torch.tensor(0.0)
+
+accelerator = SimpleNamespace(
+ distributed_type = "DistributedType.MULTI_GPU", num_processes = 2, device = torch.device("cpu"),
+ no_sync = lambda model: contextlib.nullcontext(),
+)
+trainer = SimpleNamespace(
+ training_step = training_step, get_train_dataloader = lambda: None, accelerator = accelerator,
+ args = SimpleNamespace(gradient_accumulation_steps = 1), model_accepts_loss_kwargs = True,
+ label_smoother = None,
+)
+os.environ["UNSLOTH_TOKEN_BUDGET"] = "16"
+tb.install_token_budget(trainer)
+trainer.training_step(None, batch, 100)
+print("CALLS", rank, len(calls), flush = True)
+dist.destroy_process_group()
+"""
+
+
[email protected]("other, expected", [("8,8", (1, 1)), ("4,5,3,6,2,4,6,4,6", (3, 3))])
+def test_ddp_ranks_split_only_when_every_rank_needs_the_same_count(tmp_path, other, expected):
+ import socket
+ import subprocess
+
+ with socket.socket() as s:
+ s.bind(("127.0.0.1", 0))
+ port = s.getsockname()[1]
+ env = dict(
+ __import__("os").environ,
+ MASTER_ADDR = "127.0.0.1",
+ MASTER_PORT = str(port),
+ WORLD_SIZE = "2",
+ )
+ env.pop("UNSLOTH_TOKEN_BALANCE", None)
+ procs = [
+ subprocess.Popen(
+ [sys.executable, "-c", _DDP_WORKER, str(_PATH), other],
+ env = dict(env, RANK = str(rank)),
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ text = True,
+ )
+ for rank in range(2)
+ ]
+ outputs = []
+ try:
+ for p in procs:
+ outputs.append(p.communicate(timeout = 120)[0])
+ finally:
+ for p in procs:
+ if p.poll() is None:
+ p.kill()
+ calls = {}
+ for out in outputs:
+ for line in out.splitlines():
+ if line.startswith("CALLS"):
+ _, rank, n = line.split()
+ calls[int(rank)] = int(n)
+ assert (calls.get(0), calls.get(1)) == expected, outputs
+
+
[email protected]("key", ["rope_scaling", "rope_parameters"])
[email protected]("kind", ["longrope", "dynamic"])
+def test_length_dependent_rope_steps_aside_for_split_and_balance(key, kind):
+ trainer, _ = _fake_trainer()
+ assert tb._exactness_blocker(trainer) is None
+ config = SimpleNamespace(**{key: {"rope_type": kind, "original_max_position_embeddings": 4096}})
+ trainer.model = SimpleNamespace(config = config)
+ assert kind in tb._exactness_blocker(trainer)
+ # Per layer type (transformers 5 rope_parameters).
+ nested = {"full_attention": {"rope_type": kind}, "sliding_attention": {"rope_type": "default"}}
+ trainer.model = SimpleNamespace(config = SimpleNamespace(**{key: nested}))
+ assert kind in tb._exactness_blocker(trainer)
+ trainer.model = SimpleNamespace(config = SimpleNamespace(**{key: {"rope_type": "llama3"}}))
+ assert tb._exactness_blocker(trainer) is None
+
+
+def test_balance_steps_aside_for_padded_batches(monkeypatch, capsys):
+ # Padded rows: balancing token sums spreads the long rows over every micro-batch, so the
+ # padded token count grows while the longest micro-batch, and so the peak, stays the same.
+ datasets = pytest.importorskip("datasets")
+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")
+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)
+ rng = random.Random(0)
+ lengths = [rng.choice([8, 8, 8, 600]) for _ in range(64)]
+ dataset = datasets.Dataset.from_dict({"input_ids": [[1] * n for n in lengths]})
+ bs, ga = 2, 4
+
+ def padded_tokens(padding_free, packing = False):
+ sampler = torch.utils.data.SequentialSampler(dataset)
+ inner = torch.utils.data.BatchSampler(sampler, bs, drop_last = False)
+ loader = SimpleNamespace(batch_sampler = inner, dataset = dataset)
+ trainer = SimpleNamespace(
+ training_step = None,
+ get_train_dataloader = lambda: loader,
+ args = SimpleNamespace(
+ gradient_accumulation_steps = ga,
+ average_tokens_across_devices = True,
+ padding_free = padding_free,
+ packing = packing,
+ ),
+ model_accepts_loss_kwargs = True,
+ )
+ tb.install_token_budget(trainer)
+ trainer.get_train_dataloader()
+ return trainer._unsloth_token_balance_active, sum(
+ len(b) * max(lengths[i] for i in b) for b in inner
+ )
+
+ plain = sum(
+ len(lengths[s : s + bs]) * max(lengths[s : s + bs]) for s in range(0, len(lengths), bs)
+ )
+ active, tokens = padded_tokens(padding_free = False)
+ assert active is False and tokens == plain
+ assert "batches are padded" in capsys.readouterr().out
+ assert padded_tokens(padding_free = True)[0] is True
+ assert padded_tokens(padding_free = False, packing = True)[0] is True
diff --git a/unsloth/trainer.py b/unsloth/trainer.py
index 99b598235e2..8bcc542b3fa 100644
--- a/unsloth/trainer.py
+++ b/unsloth/trainer.py
@@ -34,6 +34,7 @@ from unsloth.utils import (
enable_sample_packing,
)
from unsloth.utils.packing import _stateful_mixer_kind, patch_hybrid_linear_attention_varlen
+from unsloth.utils.token_budget import install_token_budget
from unsloth_zoo.training_utils import (
unsloth_train as _unsloth_train,
)
@@ -2036,6 +2037,8 @@ def _patch_sft_trainer_auto_packing(trl_module):
)
print(message)
+ install_token_budget(self)
+
# get_peft_model installs a pre-train forward detector for plain LoRA/vision models, but only RL
# trainers run the reset via prepare_for_training_mode, so wire it into the SFT train() path too,
# else a grad-enabled probe before train() leaves the poisoned Dynamo cache in place.
diff --git a/unsloth/utils/token_budget.py b/unsloth/utils/token_budget.py
new file mode 100644
index 00000000000..2a2a50b8999
--- /dev/null
+++ b/unsloth/utils/token_budget.py
@@ -0,0 +1,563 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Opt-in token-aware SFT micro-batches, off unless their env var is set.
+
+UNSLOTH_TOKEN_BALANCE=1 spreads each optimizer step's rows over data-parallel ranks and
+accumulation slots with equal token counts; every step keeps its rows, so num_items_in_batch
+is unchanged. UNSLOTH_TOKEN_BUDGET=<tokens> runs a larger micro-batch as several passes over
+whole sequences, each divided by the same num_items_in_batch, so they sum to the unsplit loss.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import heapq
+import inspect
+import itertools
+import os
+from collections.abc import Mapping
+
+import torch
+
+__all__ = [
+ "karmarkar_karp",
+ "balance_step_indices",
+ "TokenBalancedSampler",
+ "split_batch_by_token_budget",
+ "install_token_budget",
+]
+
+_TOKEN_BALANCE_ENV = "UNSLOTH_TOKEN_BALANCE"
+_TOKEN_BUDGET_ENV = "UNSLOTH_TOKEN_BUDGET"
+
+
+def karmarkar_karp(
+ values,
+ k,
+ equal_size = False,
+):
+ """Split indices of `values` into `k` groups with near-equal sums (largest differencing method).
+
+ With `equal_size`, `len(values)` must be a multiple of `k` and every group gets the same count.
+ Deterministic: ties break on index. Mirrors verl/utils/seqlen_balancing.py:karmarkar_karp.
+ """
+ k = int(k)
+ if k < 1:
+ raise ValueError(f"k must be >= 1, got {k}")
+ n = len(values)
+ if equal_size and n % k != 0:
+ raise ValueError(f"equal_size needs len(values) % k == 0, got {n} % {k}")
+ if k == 1:
+ return [list(range(n))]
+ order = sorted(range(n), key = lambda i: (values[i], i))
+ counter = itertools.count()
+ heap = []
+
+ def push(sets):
+ sets.sort(key = lambda s: (s[0], len(s[1]), s[1]), reverse = True)
+ heapq.heappush(heap, (-(sets[0][0] - sets[-1][0]), next(counter), sets))
+
+ if equal_size:
+ for start in range(0, n, k):
+ push([[values[i], [i]] for i in order[start : start + k]])
+ else:
+ for i in order:
+ push([[values[i], [i]]] + [[0, []] for _ in range(k - 1)])
+ if not heap:
+ return [[] for _ in range(k)]
+ while len(heap) > 1:
+ a = heapq.heappop(heap)[2]
+ b = heapq.heappop(heap)[2]
+ # Largest of one state with the smallest of the other cancels the biggest difference.
+ push([[a[j][0] + b[k - 1 - j][0], a[j][1] + b[k - 1 - j][1]] for j in range(k)])
+ return [sorted(s[1]) for s in heap[0][2]]
+
+
+def balance_step_indices(
+ indices,
+ lengths,
+ batch_size,
+ grad_accum,
+ world_size = 1,
+ cross_rank = True,
+):
+ """Reorder one optimizer step's sampler output (`world_size * grad_accum * batch_size` indices).
+
+ The layout follows accelerate's BatchSamplerShard: consecutive `batch_size` chunks are global
+ batches, global batch b goes to rank b % world_size, and rank r's g-th batch of the step is
+ global batch g * world_size + r. With `cross_rank` rows move between ranks to equalize per-rank
+ tokens; without it each rank keeps its own rows and only its accumulation slots are balanced.
+ """
+ bs, ga, ws = int(batch_size), int(grad_accum), int(world_size)
+ window = bs * ga * ws
+ if len(indices) != window:
+ raise ValueError(f"expected {window} indices, got {len(indices)}")
+ if ga * ws == 1:
+ return list(indices)
+ if cross_rank and ws > 1:
+ groups = karmarkar_karp([lengths[i] for i in indices], ws, equal_size = True)
+ rank_rows = [[indices[p] for p in group] for group in groups]
+ else:
+ rank_rows = [
+ [indices[(g * ws + r) * bs + j] for g in range(ga) for j in range(bs)]
+ for r in range(ws)
+ ]
+ out = [None] * window
+ for r, rows in enumerate(rank_rows):
+ slots = karmarkar_karp([lengths[i] for i in rows], ga, equal_size = True)
+ for g, slot in enumerate(slots):
+ base = (g * ws + r) * bs
+ for j, p in enumerate(slot):
+ out[base + j] = rows[p]
+ return out
+
+
+class TokenBalancedSampler(torch.utils.data.Sampler):
+ """Applies `balance_step_indices` to each full step window; a partial tail passes through.
+
+ `keep_first_window` passes the first window through unchanged: accelerate's BatchSamplerShard
+ (even_batches) pads an uneven tail with the first `world_size * batch_size` indices it saw.
+ """
+
+ def __init__(
+ self,
+ sampler,
+ lengths,
+ batch_size,
+ grad_accum,
+ world_size = 1,
+ cross_rank = True,
+ keep_first_window = False,
+ ):
+ self.sampler = sampler
+ self.lengths = lengths
+ self.batch_size = int(batch_size)
+ self.grad_accum = int(grad_accum)
+ self.world_size = int(world_size)
+ self.cross_rank = bool(cross_rank)
+ self.keep_first_window = bool(keep_first_window)
+
+ def __len__(self):
+ return len(self.sampler)
+
+ def __iter__(self):
+ window = self.batch_size * self.grad_accum * self.world_size
+ buffer = []
+ first = self.keep_first_window
+ for index in self.sampler:
+ buffer.append(index)
+ if len(buffer) == window and first:
+ first = False
+ yield from buffer
+ buffer = []
+ elif len(buffer) == window:
+ yield from balance_step_indices(
+ buffer,
+ self.lengths,
+ self.batch_size,
+ self.grad_accum,
+ self.world_size,
+ self.cross_rank,
+ )
+ buffer = []
+ yield from buffer
+
+ def set_epoch(self, epoch):
+ if hasattr(self.sampler, "set_epoch"):
+ self.sampler.set_epoch(epoch)
+
+ def __getattr__(self, name):
+ # accelerate and the Trainer read generator / data_source / epoch off the sampler.
+ if name == "sampler":
+ raise AttributeError(name)
+ return getattr(self.sampler, name)
+
+
+def _segments_of(batch, total):
+ lengths = batch.get("packed_seq_lengths")
+ if isinstance(lengths, torch.Tensor):
+ lengths = [int(x) for x in lengths.tolist()]
+ return lengths if sum(lengths) == total else None
+ position_ids = batch.get("position_ids")
+ if not isinstance(position_ids, torch.Tensor) or tuple(position_ids.shape) != (1, total):
+ return None
+ starts = (position_ids[0] == 0).nonzero(as_tuple = False).flatten().tolist()
+ if not starts or starts[0] != 0:
+ return None
+ return [b - a for a, b in zip(starts, starts[1:] + [total])]
+
+
+def _split_padding_free(batch, budget):
+ input_ids = batch["input_ids"]
+ total = int(input_ids.shape[1])
+ if total <= budget:
+ return None
+ segments = _segments_of(batch, total)
+ if segments is None or len(segments) < 2:
+ return None
+ for key, value in batch.items():
+ if key == "packed_seq_lengths" or not isinstance(value, torch.Tensor):
+ continue
+ if value.dim() < 2 or value.shape[0] != 1 or value.shape[1] != total:
+ return None
+ # A pairwise [1, T, T] mask or bias would need both token axes cut.
+ if total in value.shape[2:]:
+ return None
+ groups, current, used = [], [], 0
+ for length in segments:
+ if current and used + length > budget:
+ groups.append(current)
+ current, used = [], 0
+ current.append(length)
+ used += length
+ groups.append(current)
+ if len(groups) < 2:
+ return None
+ stops = list(itertools.accumulate(sum(group) for group in groups))
+ labels = batch.get("labels")
+ # The last token of a chunk loses its next-token target, exact only when that target is masked.
+ # Checked even with shift_labels: Unsloth's fast causal LM path builds its loss from labels.
+ if isinstance(labels, torch.Tensor):
+ cuts = torch.tensor(stops[:-1], device = labels.device)
+ if bool((labels[0, cuts] != -100).any()):
+ return None
+ chunks, start = [], 0
+ for group, stop in zip(groups, stops):
+ chunk = {}
+ for key, value in batch.items():
+ if key == "packed_seq_lengths":
+ chunk[key] = torch.tensor(group, dtype = value.dtype, device = value.device)
+ elif isinstance(value, torch.Tensor):
+ chunk[key] = value[:, start:stop]
+ else:
+ chunk[key] = value
+ chunks.append(chunk)
+ start = stop
+ return chunks
+
+
+def _split_rows(batch, budget):
+ input_ids = batch["input_ids"]
+ rows, width = int(input_ids.shape[0]), int(input_ids.shape[1])
+ per_chunk = max(1, budget // max(width, 1))
+ if rows <= per_chunk:
+ return None
+ for value in batch.values():
+ if isinstance(value, torch.Tensor) and (value.dim() == 0 or value.shape[0] != rows):
+ return None
+ chunks = []
+ for start in range(0, rows, per_chunk):
+ chunks.append(
+ {
+ key: value[start : start + per_chunk] if isinstance(value, torch.Tensor) else value
+ for key, value in batch.items()
+ }
+ )
+ return chunks
+
+
+# Media tensors need not have one row per text row (flattened images, patches), so their batches are never split.
+_MEDIA_KEY_PARTS = (
+ "pixel",
+ "image",
+ "video",
+ "audio",
+ "vision",
+ "input_features",
+ "input_values",
+ "grid_thw",
+ "aspect_ratio",
+ "cross_attention",
+)
+
+
+def _splittable_keys(batch):
+ for key, value in batch.items():
+ if any(part in str(key).lower() for part in _MEDIA_KEY_PARTS):
+ return False
+ # Per-sample metadata in a list would be copied whole into every chunk.
+ if isinstance(value, (list, tuple, Mapping)):
+ return False
+ return True
+
+
+def split_batch_by_token_budget(batch, budget):
+ """Split a collated text batch into sub-batches of at most `budget` tokens, else return `[batch]`.
+
+ Padding-free batches ([1, T] with sequence boundaries) split between whole sequences; padded
+ batches ([B, L]) split by rows. Anything else (vision tensors, unknown shapes) is not split.
+ """
+ if not isinstance(batch, Mapping) or budget is None or budget <= 0:
+ return [batch]
+ input_ids = batch.get("input_ids")
+ if not isinstance(input_ids, torch.Tensor) or input_ids.dim() != 2:
+ return [batch]
+ if not _splittable_keys(batch):
+ return [batch]
+ if input_ids.shape[0] == 1 and ("packed_seq_lengths" in batch or "position_ids" in batch):
+ chunks = _split_padding_free(batch, int(budget))
+ else:
+ chunks = _split_rows(batch, int(budget))
+ return chunks or [batch]
+
+
+def _env_int(name):
+ value = os.environ.get(name, "").strip()
+ if not value:
+ return 0
+ try:
+ return int(value)
+ except ValueError:
+ raise ValueError(f"Unsloth: {name} must be an integer, got {value!r}.")
+
+
+def _dataset_lengths(dataset):
+ column_names = getattr(dataset, "column_names", None)
+ if column_names is None or "input_ids" not in column_names:
+ return None
+ # Batched, so the whole token column is never materialised as Python lists at once.
+ if hasattr(dataset, "select_columns") and hasattr(dataset, "iter"):
+ lengths = []
+ for rows in dataset.select_columns(["input_ids"]).iter(batch_size = 4096):
+ lengths.extend(len(ids) for ids in rows["input_ids"])
+ return lengths
+ return [len(ids) for ids in dataset["input_ids"]]
+
+
+def _accepts_num_items(fn):
+ try:
+ return "num_items_in_batch" in inspect.signature(fn).parameters
+ except (TypeError, ValueError):
+ return False
+
+
+def _loss_uses_step_count(trainer):
+ """True when each micro-batch loss is a sum divided by the step's shared num_items_in_batch."""
+ if getattr(trainer, "compute_loss_func", None) is not None:
+ return True
+ # Without loss kwargs the Trainer counts nothing and every micro-batch loss is its own mean.
+ if not getattr(trainer, "model_accepts_loss_kwargs", False):
+ return False
+ smoother = getattr(trainer, "label_smoother", None)
+ # transformers < 5 LabelSmoother returns a per-call mean.
+ return smoother is None or _accepts_num_items(smoother.__call__)
+
+
+def _router_aux_loss_active(trainer):
+ """MoE load-balancing loss is a nonlinear per-batch statistic, so regrouping rows changes it."""
+ if getattr(trainer, "aux_loss_enabled", False):
+ return True
+ model = getattr(trainer, "model", None)
+ model = getattr(model, "module", model)
+ config = getattr(model, "config", None)
+ configs = [config]
+ if hasattr(config, "get_text_config"):
+ configs.append(config.get_text_config())
+ for cfg in configs:
+ if (
+ getattr(cfg, "output_router_logits", False)
+ and getattr(cfg, "router_aux_loss_coef", 1.0) != 0
+ ):
+ return True
+ return False
+
+
+def _exactness_blocker(trainer):
+ args = getattr(trainer, "args", None)
+ optim = str(getattr(args, "optim", "")).lower()
+ # LOMO / AdaLomo and the *_layerwise optimizers update weights inside every backward.
+ if "lomo" in optim or "layerwise" in optim:
+ return "this optimizer updates the weights on every backward"
+ # NEFTune scales its noise by the forward's sequence length, which a split shortens.
+ if getattr(args, "neftune_noise_alpha", None):
+ return "NEFTune noise depends on the sequence length of each forward"
+ if not _loss_uses_step_count(trainer):
+ return (
+ "the loss is a per-micro-batch mean (no shared num_items_in_batch, or label smoothing)"
+ )
+ if _router_aux_loss_active(trainer):
+ return "the MoE router auxiliary loss is active"
+ rope = _length_dependent_rope(trainer)
+ if rope is not None:
+ return f"{rope} RoPE picks its frequencies from each forward's length"
+ return None
+
+
+def _length_dependent_rope(trainer):
+ """LongRoPE or dynamic RoPE anywhere in the text config (flat or per layer type): both pick
+ frequencies from each forward's length, which splitting or regrouping changes."""
+ config = getattr(getattr(trainer, "model", None), "config", None)
+ config = getattr(config, "text_config", None) or config
+
+ def kinds(value):
+ if isinstance(value, Mapping):
+ kind = value.get("rope_type", value.get("type"))
+ if kind is not None:
+ yield str(kind).lower()
+ for item in value.values():
+ yield from kinds(item)
+
+ for name in ("rope_scaling", "rope_parameters"):
+ for kind in kinds(getattr(config, name, None)):
+ if kind in ("longrope", "dynamic"):
+ return kind
+ return None
+
+
+def _note_once(trainer, message):
+ notes = getattr(trainer, "_unsloth_token_budget_notes", None)
+ if notes is None:
+ notes = set()
+ trainer._unsloth_token_budget_notes = notes
+ if message not in notes:
+ notes.add(message)
+ print(message)
+
+
+def _padded_batches(trainer):
+ """True when the SFT collator pads rows: neither padding-free nor packing is on."""
+ args = getattr(trainer, "args", None)
+ return (
+ hasattr(args, "padding_free")
+ and not args.padding_free
+ and not getattr(args, "packing", False)
+ )
+
+
+def _balance_dataloader(trainer, dataloader):
+ if _exactness_blocker(trainer) is not None:
+ return False
+ # A padded micro-batch costs rows x its longest row: equal token sums put a long row in every
+ # micro-batch, which adds padding without lowering the peak.
+ if _padded_batches(trainer):
+ return False
+ # accelerate's DataLoaderDispatcher deals successive inner batches to ranks behind a plain BatchSampler.
+ if any(c.__name__ == "DataLoaderDispatcher" for c in type(dataloader).__mro__):
+ if int(getattr(getattr(dataloader, "state", None), "num_processes", 1)) > 1:
+ return False
+ batch_sampler = getattr(dataloader, "batch_sampler", None)
+ world_size = 1
+ keep_first_window = False
+ if hasattr(batch_sampler, "num_processes") and hasattr(batch_sampler, "batch_sampler"):
+ # accelerate's BatchSamplerShard with split_batches False deals whole batches round robin.
+ if getattr(batch_sampler, "split_batches", False):
+ return False
+ world_size = int(batch_sampler.num_processes)
+ keep_first_window = not getattr(batch_sampler, "drop_last", False) and getattr(
+ batch_sampler, "even_batches", True
+ )
+ batch_sampler = batch_sampler.batch_sampler
+ if not isinstance(batch_sampler, torch.utils.data.BatchSampler):
+ return False
+ sampler = batch_sampler.sampler
+ if isinstance(sampler, TokenBalancedSampler):
+ return True
+ if not isinstance(
+ sampler, (torch.utils.data.RandomSampler, torch.utils.data.SequentialSampler)
+ ):
+ # group_by_length / batch_rebalance already pick their own order.
+ return False
+ lengths = _dataset_lengths(getattr(dataloader, "dataset", None))
+ if lengths is None:
+ return False
+ if keep_first_window and len(sampler) % (world_size * batch_sampler.batch_size) == 0:
+ keep_first_window = False
+ args = trainer.args
+ # Moving rows between ranks keeps the loss exact only when the token count is global.
+ cross_rank = world_size > 1 and bool(getattr(args, "average_tokens_across_devices", False))
+ batch_sampler.sampler = TokenBalancedSampler(
+ sampler,
+ lengths,
+ batch_sampler.batch_size,
+ args.gradient_accumulation_steps,
+ world_size,
+ cross_rank,
+ keep_first_window,
+ )
+ return True
+
+
+def _same_count_on_every_rank(count, accelerator):
+ if accelerator is None or getattr(accelerator, "num_processes", 1) <= 1:
+ return True
+ import torch.distributed as dist
+
+ if not (dist.is_available() and dist.is_initialized()):
+ return True
+ bounds = torch.tensor([count, -count], dtype = torch.int64, device = accelerator.device)
+ dist.all_reduce(bounds, op = dist.ReduceOp.MAX)
+ return int(bounds[0]) == -int(bounds[1])
+
+
+def install_token_budget(trainer):
+ """Install whichever of UNSLOTH_TOKEN_BALANCE / UNSLOTH_TOKEN_BUDGET is set on `trainer`."""
+ balance = os.environ.get(_TOKEN_BALANCE_ENV, "0").lower() in ("1", "true", "yes", "on")
+ budget = _env_int(_TOKEN_BUDGET_ENV)
+ if (not balance and budget <= 0) or getattr(trainer, "_unsloth_token_budget_installed", False):
+ return
+ trainer._unsloth_token_budget_installed = True
+
+ if balance:
+ original_get_train_dataloader = trainer.get_train_dataloader
+
+ def get_train_dataloader(*args, **kwargs):
+ dataloader = original_get_train_dataloader(*args, **kwargs)
+ trainer._unsloth_token_balance_active = _balance_dataloader(trainer, dataloader)
+ if not trainer._unsloth_token_balance_active:
+ reason = _exactness_blocker(trainer)
+ if reason is None and _padded_batches(trainer):
+ reason = "batches are padded, so equal token sums would only add padding"
+ reason = reason or "unsupported sampler or dataset"
+ _note_once(trainer, f"Unsloth: UNSLOTH_TOKEN_BALANCE ignored: {reason}.")
+ return dataloader
+
+ trainer.get_train_dataloader = get_train_dataloader
+
+ if budget > 0:
+ original_training_step = trainer.training_step
+ trainer._unsloth_token_budget_splits = 0
+
+ def training_step(
+ model,
+ inputs,
+ num_items_in_batch = None,
+ *args,
+ **kwargs,
+ ):
+ accelerator = getattr(trainer, "accelerator", None)
+ distributed_type = str(getattr(accelerator, "distributed_type", "NO")).upper()
+ # Only None (loss would be a per-chunk mean), plain or DDP: DeepSpeed and FSDP count micro-steps themselves.
+ if num_items_in_batch is None or not (
+ distributed_type.endswith("NO") or distributed_type.endswith("MULTI_GPU")
+ ):
+ return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)
+ reason = _exactness_blocker(trainer)
+ if reason is not None:
+ _note_once(trainer, f"Unsloth: UNSLOTH_TOKEN_BUDGET not splitting: {reason}.")
+ return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)
+ chunks = split_batch_by_token_budget(inputs, budget)
+ if not _same_count_on_every_rank(len(chunks), accelerator):
+ # Each chunk runs compute_loss (TRL gathers metrics there) and its own backward:
+ # ranks with different counts would issue mismatched collectives and hang.
+ _note_once(
+ trainer,
+ "Unsloth: UNSLOTH_TOKEN_BUDGET not splitting steps whose ranks need different chunk counts.",
+ )
+ return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)
+ if len(chunks) == 1:
+ return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)
+ trainer._unsloth_token_budget_splits += 1
+ total = None
+ for i, chunk in enumerate(chunks):
+ context = (
+ accelerator.no_sync(model)
+ if i < len(chunks) - 1 and accelerator is not None
+ else contextlib.nullcontext()
+ )
+ with context:
+ loss = original_training_step(model, chunk, num_items_in_batch, *args, **kwargs)
+ total = loss if total is None else total + loss
+ return total
+
+ trainer.training_step = training_step