Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Opt-in token-balanced and token-budget SFT micro-batches (#13878)
Daniel Haneac949c8a0ef
diff --git a/tests/test_token_budget_batches.py b/tests/test_token_budget_batches.pynew file mode 100644index 00000000000..c7af5080be3--- /dev/null+++ b/tests/test_token_budget_batches.py@@ -0,0 +1,728 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_TOKEN_BALANCE / UNSLOTH_TOKEN_BUDGET: partitioner, step invariants, exact split loss. CPU only."""++from __future__ import annotations++import contextlib+import importlib.util+import itertools+import random+import sys+from pathlib import Path+from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")++_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "utils" / "token_budget.py"+_spec = importlib.util.spec_from_file_location("_unsloth_token_budget", _PATH)+tb = importlib.util.module_from_spec(_spec)+sys.modules[_spec.name] = tb+_spec.loader.exec_module(tb)+++def _sums(values, groups):+ return [sum(values[i] for i in g) for g in groups]+++def _best_spread(values, k):+ best = None+ for labels in itertools.product(range(k), repeat = len(values)):+ sums = [0] * k+ for v, g in zip(values, labels):+ sums[g] += v+ spread = max(sums) - min(sums)+ best = spread if best is None else min(best, spread)+ return best++[email protected]("seed", range(20))+def test_karmarkar_karp_is_a_partition_close_to_optimal(seed):+ rng = random.Random(seed)+ values = [rng.randint(1, 100) for _ in range(rng.randint(3, 9))]+ k = rng.choice([2, 3])+ groups = tb.karmarkar_karp(values, k)+ assert sorted(i for g in groups for i in g) == list(range(len(values)))+ sums = _sums(values, groups)+ assert max(sums) - min(sums) <= _best_spread(values, k) + max(values)+++def test_karmarkar_karp_equal_size_and_classic_example():+ groups = tb.karmarkar_karp([8, 7, 6, 5, 4], 2)+ assert sorted(_sums([8, 7, 6, 5, 4], groups)) == [14, 16]+ rng = random.Random(0)+ values = [rng.randint(1, 4096) for _ in range(64)]+ groups = tb.karmarkar_karp(values, 8, equal_size = True)+ assert all(len(g) == 8 for g in groups)+ assert sorted(i for g in groups for i in g) == list(range(64))+ sums = _sums(values, groups)+ assert max(sums) - min(sums) < 0.05 * (sum(values) / 8)+ with pytest.raises(ValueError):+ tb.karmarkar_karp(values[:63], 8, equal_size = True)+++def _layout(order, bs, ga, ws):+ """accelerate BatchSamplerShard: global batch b -> rank b % ws, the rank's (b // ws)-th batch."""+ batches = [order[i : i + bs] for i in range(0, len(order), bs)]+ return [[batches[g * ws + r] for g in range(ga)] for r in range(ws)]+++def _num_items(labels, batches):+ # HF / zoo count: shifted labels that are not -100, summed over the step's micro-batches.+ return sum(int((labels[i][1:] != -100).sum()) for batch in batches for i in batch)++[email protected]("ws,ga,bs", [(1, 4, 2), (2, 2, 3), (4, 2, 2), (2, 1, 4)])[email protected]("cross_rank", [True, False])+def test_step_rows_and_num_items_are_unchanged(ws, ga, bs, cross_rank):+ rng = random.Random(ws * 100 + ga * 10 + bs)+ n = ws * ga * bs+ lengths = [rng.choice([16, 32, 512, 2048]) for _ in range(n)]+ labels = []+ for length in lengths:+ row = torch.randint(0, 50, (length,))+ row[: rng.randint(0, length - 1)] = -100+ labels.append(row)+ order = list(range(n))+ rng.shuffle(order)+ balanced = tb.balance_step_indices(order, lengths, bs, ga, ws, cross_rank)+ assert sorted(balanced) == sorted(order)++ before = _layout(order, bs, ga, ws)+ after = _layout(balanced, bs, ga, ws)+ assert sum(_num_items(labels, r) for r in before) == sum(_num_items(labels, r) for r in after)+ assert all(len(b) == bs for r in after for b in r)+ if not cross_rank or ws == 1:+ for r in range(ws):+ assert sorted(sum(before[r], [])) == sorted(sum(after[r], []))+ assert _num_items(labels, before[r]) == _num_items(labels, after[r])+++def test_balancing_evens_out_rank_and_slot_tokens():+ bs, ga, ws = 4, 4, 4+ lengths = [4096] * 8 + [64] * (bs * ga * ws - 8)+ order = list(range(len(lengths)))+ before = _layout(order, bs, ga, ws)+ after = _layout(tb.balance_step_indices(order, lengths, bs, ga, ws, True), bs, ga, ws)++ def rank_tokens(layout):+ return [sum(lengths[i] for b in r for i in b) for r in layout]++ def slot_max(layout):+ return max(sum(lengths[i] for i in b) for r in layout for b in r)++ assert max(rank_tokens(before)) - min(rank_tokens(before)) > 10_000+ assert max(rank_tokens(after)) - min(rank_tokens(after)) == 0+ assert slot_max(after) < slot_max(before)+++def test_sampler_keeps_the_index_multiset_and_forwards_set_epoch():+ lengths = [random.Random(i).randint(1, 999) for i in range(23)]+ base = torch.utils.data.RandomSampler(range(23), generator = torch.Generator().manual_seed(0))+ sampler = tb.TokenBalancedSampler(base, lengths, batch_size = 2, grad_accum = 2, world_size = 2)+ out = list(sampler)+ assert len(sampler) == 23 and sorted(out) == list(range(23))+ calls = []+ base.set_epoch = calls.append+ sampler.set_epoch(3)+ assert calls == [3]+ assert sampler.generator is base.generator+++def _padding_free_batch(segments, mask_boundaries = True):+ total = sum(segments)+ torch.manual_seed(0)+ input_ids = torch.randint(1, 30, (1, total))+ labels = input_ids.clone()+ position_ids = torch.cat([torch.arange(s) for s in segments]).unsqueeze(0)+ if mask_boundaries:+ labels[position_ids == 0] = -100+ return {+ "input_ids": input_ids,+ "labels": labels,+ "position_ids": position_ids,+ "packed_seq_lengths": torch.tensor(segments, dtype = torch.int32),+ }+++class _TinyLM(torch.nn.Module):+ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ self.emb = torch.nn.Embedding(32, 8)+ self.head = torch.nn.Linear(8, 32)++ def loss(self, batch, num_items):+ logits = self.head(torch.tanh(self.emb(batch["input_ids"])))+ shift_logits = logits[:, :-1].reshape(-1, 32)+ shift_labels = batch["labels"][:, 1:].reshape(-1)+ return (+ torch.nn.functional.cross_entropy(+ shift_logits, shift_labels, ignore_index = -100, reduction = "sum"+ )+ / num_items+ )+++def _grads(model, batches, num_items):+ model.zero_grad()+ total = 0.0+ for batch in batches:+ loss = model.loss(batch, num_items)+ loss.backward()+ total += float(loss)+ return total, [p.grad.clone() for p in model.parameters()]+++def test_padding_free_split_is_exact_and_respects_the_budget():+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4])+ chunks = tb.split_batch_by_token_budget(batch, 16)+ assert len(chunks) > 1+ for chunk in chunks:+ assert chunk["input_ids"].shape[1] <= 16 or len(chunk["packed_seq_lengths"]) == 1+ assert int(chunk["packed_seq_lengths"].sum()) == chunk["input_ids"].shape[1]+ assert int(chunk["position_ids"][0, 0]) == 0+ for key in ("input_ids", "labels", "position_ids"):+ assert torch.equal(torch.cat([c[key] for c in chunks], dim = 1), batch[key])++ num_items = int((batch["labels"][:, 1:] != -100).sum())+ model = _TinyLM()+ full_loss, full_grads = _grads(model, [batch], num_items)+ split_loss, split_grads = _grads(model, chunks, num_items)+ assert abs(full_loss - split_loss) < 1e-5+ for a, b in zip(full_grads, split_grads):+ torch.testing.assert_close(a, b, atol = 1e-6, rtol = 1e-5)+++def test_split_refuses_when_a_cut_would_drop_a_real_target():+ batch = _padding_free_batch([5, 9, 3, 12], mask_boundaries = False)+ assert tb.split_batch_by_token_budget(batch, 10) == [batch]+++def test_split_leaves_small_unknown_and_vision_batches_alone():+ batch = _padding_free_batch([5, 6])+ assert tb.split_batch_by_token_budget(batch, 64) == [batch]+ vision = {"input_ids": torch.ones(4, 8, dtype = torch.long), "pixel_values": torch.ones(7, 3)}+ assert tb.split_batch_by_token_budget(vision, 8) == [vision]+++def test_padded_rows_split_is_exact():+ torch.manual_seed(1)+ input_ids = torch.randint(1, 30, (6, 10))+ labels = input_ids.clone()+ labels[:, :3] = -100+ batch = {"input_ids": input_ids, "labels": labels, "attention_mask": torch.ones_like(input_ids)}+ chunks = tb.split_batch_by_token_budget(batch, 20)+ assert [c["input_ids"].shape[0] for c in chunks] == [2, 2, 2]+ num_items = int((labels[:, 1:] != -100).sum())+ model = _TinyLM()+ full_loss, full_grads = _grads(model, [batch], num_items)+ split_loss, split_grads = _grads(model, chunks, num_items)+ assert abs(full_loss - split_loss) < 1e-5+ for a, b in zip(full_grads, split_grads):+ torch.testing.assert_close(a, b, atol = 1e-6, rtol = 1e-5)+++class _FakeAccelerator:+ distributed_type = "DistributedType.MULTI_GPU"++ def __init__(self):+ self.no_sync_calls = 0++ @contextlib.contextmanager+ def no_sync(self, model):+ self.no_sync_calls += 1+ yield+++def _fake_trainer():+ seen = []++ def training_step(+ model,+ inputs,+ num_items_in_batch = None,+ ):+ seen.append((inputs["input_ids"].shape[1], num_items_in_batch))+ return torch.tensor(float(inputs["input_ids"].shape[1]) / num_items_in_batch)++ trainer = SimpleNamespace(+ training_step = training_step,+ get_train_dataloader = lambda: None,+ accelerator = _FakeAccelerator(),+ args = SimpleNamespace(gradient_accumulation_steps = 1),+ model_accepts_loss_kwargs = True,+ label_smoother = None,+ )+ return trainer, seen+++def test_install_is_a_no_op_without_the_env(monkeypatch):+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)+ monkeypatch.delenv("UNSLOTH_TOKEN_BALANCE", raising = False)+ trainer, _ = _fake_trainer()+ step, loader = trainer.training_step, trainer.get_train_dataloader+ tb.install_token_budget(trainer)+ assert trainer.training_step is step and trainer.get_train_dataloader is loader+++def test_budgeted_training_step_sums_chunks_and_syncs_once(monkeypatch):+ monkeypatch.setenv("UNSLOTH_TOKEN_BUDGET", "16")+ monkeypatch.delenv("UNSLOTH_TOKEN_BALANCE", raising = False)+ trainer, seen = _fake_trainer()+ tb.install_token_budget(trainer)+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4])+ loss = trainer.training_step(None, batch, 100)+ assert len(seen) > 1 and all(n == 100 for _, n in seen)+ assert sum(width for width, _ in seen) == 40+ assert abs(float(loss) - 0.40) < 1e-6+ assert trainer.accelerator.no_sync_calls == len(seen) - 1+ assert trainer._unsloth_token_budget_splits == 1+ # Without a token count the loss is a per-call mean, so the batch is not split.+ seen.clear()+ with pytest.raises(TypeError):+ # The fake divides by num_items_in_batch: reaching it unsplit with None proves the pass-through.+ trainer.training_step(None, batch, None)+ assert seen == [(40, None)]+++def test_balanced_dataloader_end_to_end_with_accelerate_shards(monkeypatch):+ accelerate_data_loader = pytest.importorskip("accelerate.data_loader")+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)+ datasets = pytest.importorskip("datasets")+ rng = random.Random(0)+ lengths = [rng.choice([8, 8, 8, 600]) for _ in range(64)]+ dataset = datasets.Dataset.from_dict({"input_ids": [[1] * n for n in lengths]})+ bs, ga, ws = 2, 4, 2++ def make(rank):+ sampler = torch.utils.data.SequentialSampler(dataset)+ inner = torch.utils.data.BatchSampler(sampler, bs, drop_last = False)+ shard = accelerate_data_loader.BatchSamplerShard(+ inner, num_processes = ws, process_index = rank, split_batches = False+ )+ loader = SimpleNamespace(batch_sampler = shard, dataset = dataset)+ trainer = SimpleNamespace(+ training_step = None,+ get_train_dataloader = lambda: loader,+ args = SimpleNamespace(+ gradient_accumulation_steps = ga, average_tokens_across_devices = True+ ),+ model_accepts_loss_kwargs = True,+ )+ tb.install_token_budget(trainer)+ trainer.get_train_dataloader()+ assert trainer._unsloth_token_balance_active+ return list(shard)++ per_rank = [make(r) for r in range(ws)]+ steps = len(per_rank[0]) // ga+ for s in range(steps):+ step_rows = [i for r in range(ws) for b in per_rank[r][s * ga : (s + 1) * ga] for i in b]+ assert sorted(step_rows) == list(range(s * ga * ws * bs, (s + 1) * ga * ws * bs))+ tokens = [+ sum(lengths[i] for b in per_rank[r][s * ga : (s + 1) * ga] for i in b)+ for r in range(ws)+ ]+ assert max(tokens) - min(tokens) <= max(lengths)+++class _MeanLabelSmoother:+ # transformers < 5 LabelSmoother.__call__ signature: no num_items_in_batch, returns a per-call mean.+ def __call__(+ self,+ model_output,+ labels,+ shift_labels = False,+ ):+ raise AssertionError("not called")+++class _CountLabelSmoother:+ def __call__(+ self,+ model_output,+ labels,+ shift_labels = False,+ num_items_in_batch = None,+ ):+ raise AssertionError("not called")+++def _moe_model(coef):+ config = SimpleNamespace(output_router_logits = True, router_aux_loss_coef = coef)+ return SimpleNamespace(config = config)+++ "override,splits",+ [+ ({}, True),+ ({"label_smoother": _CountLabelSmoother()}, True),+ ({"label_smoother": _MeanLabelSmoother()}, False),+ ({"model_accepts_loss_kwargs": False}, False),+ ({"aux_loss_enabled": True}, False),+ ({"model": _moe_model(0.01)}, False),+ ({"model": _moe_model(0.0)}, True),+ ],+)+def test_split_steps_aside_when_chunk_losses_do_not_sum_exactly(+ monkeypatch, capsys, override, splits+):+ monkeypatch.setenv("UNSLOTH_TOKEN_BUDGET", "16")+ monkeypatch.delenv("UNSLOTH_TOKEN_BALANCE", raising = False)+ trainer, seen = _fake_trainer()+ for key, value in override.items():+ setattr(trainer, key, value)+ tb.install_token_budget(trainer)+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4])+ trainer.training_step(None, batch, 100)+ trainer.training_step(None, batch, 100)+ assert (len(seen) > 2) == splits+ if not splits:+ assert seen == [(40, 100), (40, 100)]+ assert capsys.readouterr().out.count("UNSLOTH_TOKEN_BUDGET not splitting") == 1+++def test_split_refuses_media_keys_and_per_sample_metadata():+ rows = {+ "input_ids": torch.ones(4, 8, dtype = torch.long),+ "labels": torch.ones(4, 8, dtype = torch.long),+ }+ assert len(tb.split_batch_by_token_budget(rows, 8)) == 4+ # Flattened images that happen to total B rows still must not be paired row by row.+ for extra in (+ {"pixel_values": torch.ones(4, 3)},+ {"image_grid_thw": torch.ones(4, 3, dtype = torch.long)},+ {"input_values": torch.ones(4, 16000)},+ {"num_images": [0, 2, 1, 1]},+ ):+ batch = {**rows, **extra}+ assert tb.split_batch_by_token_budget(batch, 8) == [batch]+ packed = {**_padding_free_batch([5, 9, 3, 12]), "pixel_values": torch.ones(1, 29, 3)}+ assert tb.split_batch_by_token_budget(packed, 10) == [packed]+++def _shard_steps(n, bs, ga, ws, balance, **trainer_kwargs):+ accelerate_data_loader = pytest.importorskip("accelerate.data_loader")+ datasets = pytest.importorskip("datasets")+ rng = random.Random(1)+ lengths = [rng.choice([8, 8, 8, 600]) for _ in range(n)]+ dataset = datasets.Dataset.from_dict({"input_ids": [[1] * k for k in lengths]})+ per_rank, active = [], []+ for rank in range(ws):+ base = torch.utils.data.RandomSampler(dataset, generator = torch.Generator().manual_seed(3))+ inner = torch.utils.data.BatchSampler(base, bs, drop_last = False)+ shard = accelerate_data_loader.BatchSamplerShard(+ inner, num_processes = ws, process_index = rank, split_batches = False, even_batches = True+ )+ if balance:+ loader = SimpleNamespace(batch_sampler = shard, dataset = dataset)+ kwargs = {"model_accepts_loss_kwargs": True, **trainer_kwargs}+ optim = kwargs.pop("args_optim", "adamw_torch")+ trainer = SimpleNamespace(+ training_step = None,+ get_train_dataloader = lambda: loader,+ args = SimpleNamespace(+ gradient_accumulation_steps = ga, average_tokens_across_devices = True, optim = optim+ ),+ **kwargs,+ )+ tb.install_token_budget(trainer)+ trainer.get_train_dataloader()+ active.append(trainer._unsloth_token_balance_active)+ per_rank.append(list(shard))+ steps = []+ for s in range(-(-len(per_rank[0]) // ga)):+ steps.append(+ sorted(i for r in range(ws) for b in per_rank[r][s * ga : (s + 1) * ga] for i in b)+ )+ return steps, active++[email protected]("n", [61, 64, 67])+def test_balance_keeps_every_step_including_the_even_batches_tail(monkeypatch, n):+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)+ bs, ga, ws = 2, 4, 2+ plain, _ = _shard_steps(n, bs, ga, ws, balance = False)+ balanced, active = _shard_steps(n, bs, ga, ws, balance = True)+ assert all(active)+ assert balanced == plain+++ "override",+ [+ {"model_accepts_loss_kwargs": False},+ {"label_smoother": _MeanLabelSmoother()},+ {"aux_loss_enabled": True},+ ],+)+def test_balance_steps_aside_when_slot_grouping_changes_the_loss(monkeypatch, capsys, override):+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)+ plain, _ = _shard_steps(64, 2, 4, 1, balance = False)+ balanced, active = _shard_steps(64, 2, 4, 1, balance = True, **override)+ assert active == [False] and balanced == plain+ assert "UNSLOTH_TOKEN_BALANCE ignored" in capsys.readouterr().out+++def test_real_label_smoother_is_classified_by_its_signature():+ trainer_pt_utils = pytest.importorskip("transformers.trainer_pt_utils")+ import transformers++ smoother = trainer_pt_utils.LabelSmoother(epsilon = 0.1)+ trainer = SimpleNamespace(model_accepts_loss_kwargs = True, label_smoother = smoother)+ uses_count = tb._loss_uses_step_count(trainer)+ if int(transformers.__version__.split(".")[0]) < 5:+ assert not uses_count+ assert uses_count == (+ "num_items_in_batch" in __import__("inspect").signature(smoother.__call__).parameters+ )+++def test_split_refuses_pairwise_masks():+ batch = _padding_free_batch([5, 9, 3, 12])+ total = batch["input_ids"].shape[1]+ batch["attention_mask"] = torch.ones(1, total, total, dtype = torch.bool).tril()+ assert tb.split_batch_by_token_budget(batch, 10) == [batch]+++class _CountingTensor(torch.Tensor):+ host_reads = 0++ @classmethod+ def __torch_function__(+ cls,+ func,+ types,+ args = (),+ kwargs = None,+ ):+ if func in (+ torch.Tensor.__int__,+ torch.Tensor.item,+ torch.Tensor.tolist,+ torch.Tensor.__bool__,+ ):+ cls.host_reads += 1+ return super().__torch_function__(func, types, args, kwargs or {})+++def test_split_reads_boundary_labels_in_one_transfer():+ batch = _padding_free_batch([3] * 40)+ batch["labels"] = batch["labels"].as_subclass(_CountingTensor)+ _CountingTensor.host_reads = 0+ chunks = tb.split_batch_by_token_budget(batch, 6)+ assert len(chunks) == 20+ assert _CountingTensor.host_reads <= 1+++def test_lomo_steps_aside_for_split_and_balance(monkeypatch, capsys):+ monkeypatch.setenv("UNSLOTH_TOKEN_BUDGET", "16")+ monkeypatch.delenv("UNSLOTH_TOKEN_BALANCE", raising = False)+ for optim, neftune in (+ ("lomo", None),+ ("adalomo", None),+ ("galore_adamw_layerwise", None),+ ("adamw_torch", 5.0),+ ):+ trainer, seen = _fake_trainer()+ trainer.args.optim = optim+ trainer.args.neftune_noise_alpha = neftune+ tb.install_token_budget(trainer)+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4])+ trainer.training_step(None, batch, 100)+ trainer.training_step(None, batch, 100)+ assert seen == [(40, 100), (40, 100)]+ assert capsys.readouterr().out.count("UNSLOTH_TOKEN_BUDGET not splitting") == 1+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)+ plain, _ = _shard_steps(64, 2, 4, 1, balance = False)+ balanced, active = _shard_steps(64, 2, 4, 1, balance = True, args_optim = "lomo")+ assert active == [False] and balanced == plain+++def test_balance_steps_aside_for_multi_process_dispatched_dataloaders(monkeypatch, capsys):+ accelerate_data_loader = pytest.importorskip("accelerate.data_loader")+ datasets = pytest.importorskip("datasets")+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)+ dataset = datasets.Dataset.from_dict({"input_ids": [[1] * (1 + i % 7) for i in range(32)]})+ loader = accelerate_data_loader.DataLoaderDispatcher(dataset, batch_size = 2)+ loader.state = SimpleNamespace(num_processes = 2)+ trainer = SimpleNamespace(+ training_step = None,+ get_train_dataloader = lambda: loader,+ args = SimpleNamespace(gradient_accumulation_steps = 4, average_tokens_across_devices = True),+ model_accepts_loss_kwargs = True,+ )+ tb.install_token_budget(trainer)+ trainer.get_train_dataloader()+ assert trainer._unsloth_token_balance_active is False+ assert not isinstance(loader.batch_sampler.sampler, tb.TokenBalancedSampler)+ assert "UNSLOTH_TOKEN_BALANCE ignored" in capsys.readouterr().out+++def test_split_checks_boundary_labels_even_with_shift_labels():+ # Unsloth's fast causal LM path builds its loss from labels, not shift_labels.+ batch = _padding_free_batch([5, 9, 3, 12, 7, 4], mask_boundaries = False)+ batch["shift_labels"] = torch.roll(batch["labels"], -1, dims = 1)+ assert len(tb.split_batch_by_token_budget(batch, 16)) == 1+++def test_dataset_lengths_are_read_in_batches():+ datasets = pytest.importorskip("datasets")+ dataset = datasets.Dataset.from_dict(+ {"input_ids": [[1] * (1 + i % 7) for i in range(10_000)], "x": [0] * 10_000}+ )+ dataset = dataset.shuffle(seed = 0)+ assert tb._dataset_lengths(dataset) == [len(ids) for ids in dataset["input_ids"]]+++_DDP_WORKER = r"""+import contextlib, importlib.util, os, sys+from types import SimpleNamespace+import torch, torch.distributed as dist++spec = importlib.util.spec_from_file_location("_tb", sys.argv[1])+tb = importlib.util.module_from_spec(spec)+spec.loader.exec_module(tb)+dist.init_process_group("gloo")+rank = dist.get_rank()+segments = [[5, 9, 3, 12, 7, 4], [int(x) for x in sys.argv[2].split(",")]][rank]+total = sum(segments)+input_ids = torch.randint(1, 30, (1, total))+position_ids = torch.cat([torch.arange(s) for s in segments]).unsqueeze(0)+labels = input_ids.clone()+labels[position_ids == 0] = -100+batch = dict(input_ids = input_ids, labels = labels, position_ids = position_ids,+ packed_seq_lengths = torch.tensor(segments, dtype = torch.int32))+calls = []++def training_step(model, inputs, num_items_in_batch = None):+ calls.append(inputs["input_ids"].shape[1])+ # Stands in for TRL's per-compute_loss metric gather: one collective per call.+ dist.all_reduce(torch.ones(1))+ return torch.tensor(0.0)++accelerator = SimpleNamespace(+ distributed_type = "DistributedType.MULTI_GPU", num_processes = 2, device = torch.device("cpu"),+ no_sync = lambda model: contextlib.nullcontext(),+)+trainer = SimpleNamespace(+ training_step = training_step, get_train_dataloader = lambda: None, accelerator = accelerator,+ args = SimpleNamespace(gradient_accumulation_steps = 1), model_accepts_loss_kwargs = True,+ label_smoother = None,+)+os.environ["UNSLOTH_TOKEN_BUDGET"] = "16"+tb.install_token_budget(trainer)+trainer.training_step(None, batch, 100)+print("CALLS", rank, len(calls), flush = True)+dist.destroy_process_group()+"""++[email protected]("other, expected", [("8,8", (1, 1)), ("4,5,3,6,2,4,6,4,6", (3, 3))])+def test_ddp_ranks_split_only_when_every_rank_needs_the_same_count(tmp_path, other, expected):+ import socket+ import subprocess++ with socket.socket() as s:+ s.bind(("127.0.0.1", 0))+ port = s.getsockname()[1]+ env = dict(+ __import__("os").environ,+ MASTER_ADDR = "127.0.0.1",+ MASTER_PORT = str(port),+ WORLD_SIZE = "2",+ )+ env.pop("UNSLOTH_TOKEN_BALANCE", None)+ procs = [+ subprocess.Popen(+ [sys.executable, "-c", _DDP_WORKER, str(_PATH), other],+ env = dict(env, RANK = str(rank)),+ stdout = subprocess.PIPE,+ stderr = subprocess.STDOUT,+ text = True,+ )+ for rank in range(2)+ ]+ outputs = []+ try:+ for p in procs:+ outputs.append(p.communicate(timeout = 120)[0])+ finally:+ for p in procs:+ if p.poll() is None:+ p.kill()+ calls = {}+ for out in outputs:+ for line in out.splitlines():+ if line.startswith("CALLS"):+ _, rank, n = line.split()+ calls[int(rank)] = int(n)+ assert (calls.get(0), calls.get(1)) == expected, outputs++[email protected]("key", ["rope_scaling", "rope_parameters"])[email protected]("kind", ["longrope", "dynamic"])+def test_length_dependent_rope_steps_aside_for_split_and_balance(key, kind):+ trainer, _ = _fake_trainer()+ assert tb._exactness_blocker(trainer) is None+ config = SimpleNamespace(**{key: {"rope_type": kind, "original_max_position_embeddings": 4096}})+ trainer.model = SimpleNamespace(config = config)+ assert kind in tb._exactness_blocker(trainer)+ # Per layer type (transformers 5 rope_parameters).+ nested = {"full_attention": {"rope_type": kind}, "sliding_attention": {"rope_type": "default"}}+ trainer.model = SimpleNamespace(config = SimpleNamespace(**{key: nested}))+ assert kind in tb._exactness_blocker(trainer)+ trainer.model = SimpleNamespace(config = SimpleNamespace(**{key: {"rope_type": "llama3"}}))+ assert tb._exactness_blocker(trainer) is None+++def test_balance_steps_aside_for_padded_batches(monkeypatch, capsys):+ # Padded rows: balancing token sums spreads the long rows over every micro-batch, so the+ # padded token count grows while the longest micro-batch, and so the peak, stays the same.+ datasets = pytest.importorskip("datasets")+ monkeypatch.setenv("UNSLOTH_TOKEN_BALANCE", "1")+ monkeypatch.delenv("UNSLOTH_TOKEN_BUDGET", raising = False)+ rng = random.Random(0)+ lengths = [rng.choice([8, 8, 8, 600]) for _ in range(64)]+ dataset = datasets.Dataset.from_dict({"input_ids": [[1] * n for n in lengths]})+ bs, ga = 2, 4++ def padded_tokens(padding_free, packing = False):+ sampler = torch.utils.data.SequentialSampler(dataset)+ inner = torch.utils.data.BatchSampler(sampler, bs, drop_last = False)+ loader = SimpleNamespace(batch_sampler = inner, dataset = dataset)+ trainer = SimpleNamespace(+ training_step = None,+ get_train_dataloader = lambda: loader,+ args = SimpleNamespace(+ gradient_accumulation_steps = ga,+ average_tokens_across_devices = True,+ padding_free = padding_free,+ packing = packing,+ ),+ model_accepts_loss_kwargs = True,+ )+ tb.install_token_budget(trainer)+ trainer.get_train_dataloader()+ return trainer._unsloth_token_balance_active, sum(+ len(b) * max(lengths[i] for i in b) for b in inner+ )++ plain = sum(+ len(lengths[s : s + bs]) * max(lengths[s : s + bs]) for s in range(0, len(lengths), bs)+ )+ active, tokens = padded_tokens(padding_free = False)+ assert active is False and tokens == plain+ assert "batches are padded" in capsys.readouterr().out+ assert padded_tokens(padding_free = True)[0] is True+ assert padded_tokens(padding_free = False, packing = True)[0] is Truediff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 99b598235e2..8bcc542b3fa 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -34,6 +34,7 @@ from unsloth.utils import (enable_sample_packing,)from unsloth.utils.packing import _stateful_mixer_kind, patch_hybrid_linear_attention_varlen+from unsloth.utils.token_budget import install_token_budgetfrom unsloth_zoo.training_utils import (unsloth_train as _unsloth_train,)@@ -2036,6 +2037,8 @@ def _patch_sft_trainer_auto_packing(trl_module):)print(message)+ install_token_budget(self)+# get_peft_model installs a pre-train forward detector for plain LoRA/vision models, but only RL# trainers run the reset via prepare_for_training_mode, so wire it into the SFT train() path too,# else a grad-enabled probe before train() leaves the poisoned Dynamo cache in place.diff --git a/unsloth/utils/token_budget.py b/unsloth/utils/token_budget.pynew file mode 100644index 00000000000..2a2a50b8999--- /dev/null+++ b/unsloth/utils/token_budget.py@@ -0,0 +1,563 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Opt-in token-aware SFT micro-batches, off unless their env var is set.++UNSLOTH_TOKEN_BALANCE=1 spreads each optimizer step's rows over data-parallel ranks and+accumulation slots with equal token counts; every step keeps its rows, so num_items_in_batch+is unchanged. UNSLOTH_TOKEN_BUDGET=<tokens> runs a larger micro-batch as several passes over+whole sequences, each divided by the same num_items_in_batch, so they sum to the unsplit loss.+"""++from __future__ import annotations++import contextlib+import heapq+import inspect+import itertools+import os+from collections.abc import Mapping++import torch++__all__ = [+ "karmarkar_karp",+ "balance_step_indices",+ "TokenBalancedSampler",+ "split_batch_by_token_budget",+ "install_token_budget",+]++_TOKEN_BALANCE_ENV = "UNSLOTH_TOKEN_BALANCE"+_TOKEN_BUDGET_ENV = "UNSLOTH_TOKEN_BUDGET"+++def karmarkar_karp(+ values,+ k,+ equal_size = False,+):+ """Split indices of `values` into `k` groups with near-equal sums (largest differencing method).++ With `equal_size`, `len(values)` must be a multiple of `k` and every group gets the same count.+ Deterministic: ties break on index. Mirrors verl/utils/seqlen_balancing.py:karmarkar_karp.+ """+ k = int(k)+ if k < 1:+ raise ValueError(f"k must be >= 1, got {k}")+ n = len(values)+ if equal_size and n % k != 0:+ raise ValueError(f"equal_size needs len(values) % k == 0, got {n} % {k}")+ if k == 1:+ return [list(range(n))]+ order = sorted(range(n), key = lambda i: (values[i], i))+ counter = itertools.count()+ heap = []++ def push(sets):+ sets.sort(key = lambda s: (s[0], len(s[1]), s[1]), reverse = True)+ heapq.heappush(heap, (-(sets[0][0] - sets[-1][0]), next(counter), sets))++ if equal_size:+ for start in range(0, n, k):+ push([[values[i], [i]] for i in order[start : start + k]])+ else:+ for i in order:+ push([[values[i], [i]]] + [[0, []] for _ in range(k - 1)])+ if not heap:+ return [[] for _ in range(k)]+ while len(heap) > 1:+ a = heapq.heappop(heap)[2]+ b = heapq.heappop(heap)[2]+ # Largest of one state with the smallest of the other cancels the biggest difference.+ push([[a[j][0] + b[k - 1 - j][0], a[j][1] + b[k - 1 - j][1]] for j in range(k)])+ return [sorted(s[1]) for s in heap[0][2]]+++def balance_step_indices(+ indices,+ lengths,+ batch_size,+ grad_accum,+ world_size = 1,+ cross_rank = True,+):+ """Reorder one optimizer step's sampler output (`world_size * grad_accum * batch_size` indices).++ The layout follows accelerate's BatchSamplerShard: consecutive `batch_size` chunks are global+ batches, global batch b goes to rank b % world_size, and rank r's g-th batch of the step is+ global batch g * world_size + r. With `cross_rank` rows move between ranks to equalize per-rank+ tokens; without it each rank keeps its own rows and only its accumulation slots are balanced.+ """+ bs, ga, ws = int(batch_size), int(grad_accum), int(world_size)+ window = bs * ga * ws+ if len(indices) != window:+ raise ValueError(f"expected {window} indices, got {len(indices)}")+ if ga * ws == 1:+ return list(indices)+ if cross_rank and ws > 1:+ groups = karmarkar_karp([lengths[i] for i in indices], ws, equal_size = True)+ rank_rows = [[indices[p] for p in group] for group in groups]+ else:+ rank_rows = [+ [indices[(g * ws + r) * bs + j] for g in range(ga) for j in range(bs)]+ for r in range(ws)+ ]+ out = [None] * window+ for r, rows in enumerate(rank_rows):+ slots = karmarkar_karp([lengths[i] for i in rows], ga, equal_size = True)+ for g, slot in enumerate(slots):+ base = (g * ws + r) * bs+ for j, p in enumerate(slot):+ out[base + j] = rows[p]+ return out+++class TokenBalancedSampler(torch.utils.data.Sampler):+ """Applies `balance_step_indices` to each full step window; a partial tail passes through.++ `keep_first_window` passes the first window through unchanged: accelerate's BatchSamplerShard+ (even_batches) pads an uneven tail with the first `world_size * batch_size` indices it saw.+ """++ def __init__(+ self,+ sampler,+ lengths,+ batch_size,+ grad_accum,+ world_size = 1,+ cross_rank = True,+ keep_first_window = False,+ ):+ self.sampler = sampler+ self.lengths = lengths+ self.batch_size = int(batch_size)+ self.grad_accum = int(grad_accum)+ self.world_size = int(world_size)+ self.cross_rank = bool(cross_rank)+ self.keep_first_window = bool(keep_first_window)++ def __len__(self):+ return len(self.sampler)++ def __iter__(self):+ window = self.batch_size * self.grad_accum * self.world_size+ buffer = []+ first = self.keep_first_window+ for index in self.sampler:+ buffer.append(index)+ if len(buffer) == window and first:+ first = False+ yield from buffer+ buffer = []+ elif len(buffer) == window:+ yield from balance_step_indices(+ buffer,+ self.lengths,+ self.batch_size,+ self.grad_accum,+ self.world_size,+ self.cross_rank,+ )+ buffer = []+ yield from buffer++ def set_epoch(self, epoch):+ if hasattr(self.sampler, "set_epoch"):+ self.sampler.set_epoch(epoch)++ def __getattr__(self, name):+ # accelerate and the Trainer read generator / data_source / epoch off the sampler.+ if name == "sampler":+ raise AttributeError(name)+ return getattr(self.sampler, name)+++def _segments_of(batch, total):+ lengths = batch.get("packed_seq_lengths")+ if isinstance(lengths, torch.Tensor):+ lengths = [int(x) for x in lengths.tolist()]+ return lengths if sum(lengths) == total else None+ position_ids = batch.get("position_ids")+ if not isinstance(position_ids, torch.Tensor) or tuple(position_ids.shape) != (1, total):+ return None+ starts = (position_ids[0] == 0).nonzero(as_tuple = False).flatten().tolist()+ if not starts or starts[0] != 0:+ return None+ return [b - a for a, b in zip(starts, starts[1:] + [total])]+++def _split_padding_free(batch, budget):+ input_ids = batch["input_ids"]+ total = int(input_ids.shape[1])+ if total <= budget:+ return None+ segments = _segments_of(batch, total)+ if segments is None or len(segments) < 2:+ return None+ for key, value in batch.items():+ if key == "packed_seq_lengths" or not isinstance(value, torch.Tensor):+ continue+ if value.dim() < 2 or value.shape[0] != 1 or value.shape[1] != total:+ return None+ # A pairwise [1, T, T] mask or bias would need both token axes cut.+ if total in value.shape[2:]:+ return None+ groups, current, used = [], [], 0+ for length in segments:+ if current and used + length > budget:+ groups.append(current)+ current, used = [], 0+ current.append(length)+ used += length+ groups.append(current)+ if len(groups) < 2:+ return None+ stops = list(itertools.accumulate(sum(group) for group in groups))+ labels = batch.get("labels")+ # The last token of a chunk loses its next-token target, exact only when that target is masked.+ # Checked even with shift_labels: Unsloth's fast causal LM path builds its loss from labels.+ if isinstance(labels, torch.Tensor):+ cuts = torch.tensor(stops[:-1], device = labels.device)+ if bool((labels[0, cuts] != -100).any()):+ return None+ chunks, start = [], 0+ for group, stop in zip(groups, stops):+ chunk = {}+ for key, value in batch.items():+ if key == "packed_seq_lengths":+ chunk[key] = torch.tensor(group, dtype = value.dtype, device = value.device)+ elif isinstance(value, torch.Tensor):+ chunk[key] = value[:, start:stop]+ else:+ chunk[key] = value+ chunks.append(chunk)+ start = stop+ return chunks+++def _split_rows(batch, budget):+ input_ids = batch["input_ids"]+ rows, width = int(input_ids.shape[0]), int(input_ids.shape[1])+ per_chunk = max(1, budget // max(width, 1))+ if rows <= per_chunk:+ return None+ for value in batch.values():+ if isinstance(value, torch.Tensor) and (value.dim() == 0 or value.shape[0] != rows):+ return None+ chunks = []+ for start in range(0, rows, per_chunk):+ chunks.append(+ {+ key: value[start : start + per_chunk] if isinstance(value, torch.Tensor) else value+ for key, value in batch.items()+ }+ )+ return chunks+++# Media tensors need not have one row per text row (flattened images, patches), so their batches are never split.+_MEDIA_KEY_PARTS = (+ "pixel",+ "image",+ "video",+ "audio",+ "vision",+ "input_features",+ "input_values",+ "grid_thw",+ "aspect_ratio",+ "cross_attention",+)+++def _splittable_keys(batch):+ for key, value in batch.items():+ if any(part in str(key).lower() for part in _MEDIA_KEY_PARTS):+ return False+ # Per-sample metadata in a list would be copied whole into every chunk.+ if isinstance(value, (list, tuple, Mapping)):+ return False+ return True+++def split_batch_by_token_budget(batch, budget):+ """Split a collated text batch into sub-batches of at most `budget` tokens, else return `[batch]`.++ Padding-free batches ([1, T] with sequence boundaries) split between whole sequences; padded+ batches ([B, L]) split by rows. Anything else (vision tensors, unknown shapes) is not split.+ """+ if not isinstance(batch, Mapping) or budget is None or budget <= 0:+ return [batch]+ input_ids = batch.get("input_ids")+ if not isinstance(input_ids, torch.Tensor) or input_ids.dim() != 2:+ return [batch]+ if not _splittable_keys(batch):+ return [batch]+ if input_ids.shape[0] == 1 and ("packed_seq_lengths" in batch or "position_ids" in batch):+ chunks = _split_padding_free(batch, int(budget))+ else:+ chunks = _split_rows(batch, int(budget))+ return chunks or [batch]+++def _env_int(name):+ value = os.environ.get(name, "").strip()+ if not value:+ return 0+ try:+ return int(value)+ except ValueError:+ raise ValueError(f"Unsloth: {name} must be an integer, got {value!r}.")+++def _dataset_lengths(dataset):+ column_names = getattr(dataset, "column_names", None)+ if column_names is None or "input_ids" not in column_names:+ return None+ # Batched, so the whole token column is never materialised as Python lists at once.+ if hasattr(dataset, "select_columns") and hasattr(dataset, "iter"):+ lengths = []+ for rows in dataset.select_columns(["input_ids"]).iter(batch_size = 4096):+ lengths.extend(len(ids) for ids in rows["input_ids"])+ return lengths+ return [len(ids) for ids in dataset["input_ids"]]+++def _accepts_num_items(fn):+ try:+ return "num_items_in_batch" in inspect.signature(fn).parameters+ except (TypeError, ValueError):+ return False+++def _loss_uses_step_count(trainer):+ """True when each micro-batch loss is a sum divided by the step's shared num_items_in_batch."""+ if getattr(trainer, "compute_loss_func", None) is not None:+ return True+ # Without loss kwargs the Trainer counts nothing and every micro-batch loss is its own mean.+ if not getattr(trainer, "model_accepts_loss_kwargs", False):+ return False+ smoother = getattr(trainer, "label_smoother", None)+ # transformers < 5 LabelSmoother returns a per-call mean.+ return smoother is None or _accepts_num_items(smoother.__call__)+++def _router_aux_loss_active(trainer):+ """MoE load-balancing loss is a nonlinear per-batch statistic, so regrouping rows changes it."""+ if getattr(trainer, "aux_loss_enabled", False):+ return True+ model = getattr(trainer, "model", None)+ model = getattr(model, "module", model)+ config = getattr(model, "config", None)+ configs = [config]+ if hasattr(config, "get_text_config"):+ configs.append(config.get_text_config())+ for cfg in configs:+ if (+ getattr(cfg, "output_router_logits", False)+ and getattr(cfg, "router_aux_loss_coef", 1.0) != 0+ ):+ return True+ return False+++def _exactness_blocker(trainer):+ args = getattr(trainer, "args", None)+ optim = str(getattr(args, "optim", "")).lower()+ # LOMO / AdaLomo and the *_layerwise optimizers update weights inside every backward.+ if "lomo" in optim or "layerwise" in optim:+ return "this optimizer updates the weights on every backward"+ # NEFTune scales its noise by the forward's sequence length, which a split shortens.+ if getattr(args, "neftune_noise_alpha", None):+ return "NEFTune noise depends on the sequence length of each forward"+ if not _loss_uses_step_count(trainer):+ return (+ "the loss is a per-micro-batch mean (no shared num_items_in_batch, or label smoothing)"+ )+ if _router_aux_loss_active(trainer):+ return "the MoE router auxiliary loss is active"+ rope = _length_dependent_rope(trainer)+ if rope is not None:+ return f"{rope} RoPE picks its frequencies from each forward's length"+ return None+++def _length_dependent_rope(trainer):+ """LongRoPE or dynamic RoPE anywhere in the text config (flat or per layer type): both pick+ frequencies from each forward's length, which splitting or regrouping changes."""+ config = getattr(getattr(trainer, "model", None), "config", None)+ config = getattr(config, "text_config", None) or config++ def kinds(value):+ if isinstance(value, Mapping):+ kind = value.get("rope_type", value.get("type"))+ if kind is not None:+ yield str(kind).lower()+ for item in value.values():+ yield from kinds(item)++ for name in ("rope_scaling", "rope_parameters"):+ for kind in kinds(getattr(config, name, None)):+ if kind in ("longrope", "dynamic"):+ return kind+ return None+++def _note_once(trainer, message):+ notes = getattr(trainer, "_unsloth_token_budget_notes", None)+ if notes is None:+ notes = set()+ trainer._unsloth_token_budget_notes = notes+ if message not in notes:+ notes.add(message)+ print(message)+++def _padded_batches(trainer):+ """True when the SFT collator pads rows: neither padding-free nor packing is on."""+ args = getattr(trainer, "args", None)+ return (+ hasattr(args, "padding_free")+ and not args.padding_free+ and not getattr(args, "packing", False)+ )+++def _balance_dataloader(trainer, dataloader):+ if _exactness_blocker(trainer) is not None:+ return False+ # A padded micro-batch costs rows x its longest row: equal token sums put a long row in every+ # micro-batch, which adds padding without lowering the peak.+ if _padded_batches(trainer):+ return False+ # accelerate's DataLoaderDispatcher deals successive inner batches to ranks behind a plain BatchSampler.+ if any(c.__name__ == "DataLoaderDispatcher" for c in type(dataloader).__mro__):+ if int(getattr(getattr(dataloader, "state", None), "num_processes", 1)) > 1:+ return False+ batch_sampler = getattr(dataloader, "batch_sampler", None)+ world_size = 1+ keep_first_window = False+ if hasattr(batch_sampler, "num_processes") and hasattr(batch_sampler, "batch_sampler"):+ # accelerate's BatchSamplerShard with split_batches False deals whole batches round robin.+ if getattr(batch_sampler, "split_batches", False):+ return False+ world_size = int(batch_sampler.num_processes)+ keep_first_window = not getattr(batch_sampler, "drop_last", False) and getattr(+ batch_sampler, "even_batches", True+ )+ batch_sampler = batch_sampler.batch_sampler+ if not isinstance(batch_sampler, torch.utils.data.BatchSampler):+ return False+ sampler = batch_sampler.sampler+ if isinstance(sampler, TokenBalancedSampler):+ return True+ if not isinstance(+ sampler, (torch.utils.data.RandomSampler, torch.utils.data.SequentialSampler)+ ):+ # group_by_length / batch_rebalance already pick their own order.+ return False+ lengths = _dataset_lengths(getattr(dataloader, "dataset", None))+ if lengths is None:+ return False+ if keep_first_window and len(sampler) % (world_size * batch_sampler.batch_size) == 0:+ keep_first_window = False+ args = trainer.args+ # Moving rows between ranks keeps the loss exact only when the token count is global.+ cross_rank = world_size > 1 and bool(getattr(args, "average_tokens_across_devices", False))+ batch_sampler.sampler = TokenBalancedSampler(+ sampler,+ lengths,+ batch_sampler.batch_size,+ args.gradient_accumulation_steps,+ world_size,+ cross_rank,+ keep_first_window,+ )+ return True+++def _same_count_on_every_rank(count, accelerator):+ if accelerator is None or getattr(accelerator, "num_processes", 1) <= 1:+ return True+ import torch.distributed as dist++ if not (dist.is_available() and dist.is_initialized()):+ return True+ bounds = torch.tensor([count, -count], dtype = torch.int64, device = accelerator.device)+ dist.all_reduce(bounds, op = dist.ReduceOp.MAX)+ return int(bounds[0]) == -int(bounds[1])+++def install_token_budget(trainer):+ """Install whichever of UNSLOTH_TOKEN_BALANCE / UNSLOTH_TOKEN_BUDGET is set on `trainer`."""+ balance = os.environ.get(_TOKEN_BALANCE_ENV, "0").lower() in ("1", "true", "yes", "on")+ budget = _env_int(_TOKEN_BUDGET_ENV)+ if (not balance and budget <= 0) or getattr(trainer, "_unsloth_token_budget_installed", False):+ return+ trainer._unsloth_token_budget_installed = True++ if balance:+ original_get_train_dataloader = trainer.get_train_dataloader++ def get_train_dataloader(*args, **kwargs):+ dataloader = original_get_train_dataloader(*args, **kwargs)+ trainer._unsloth_token_balance_active = _balance_dataloader(trainer, dataloader)+ if not trainer._unsloth_token_balance_active:+ reason = _exactness_blocker(trainer)+ if reason is None and _padded_batches(trainer):+ reason = "batches are padded, so equal token sums would only add padding"+ reason = reason or "unsupported sampler or dataset"+ _note_once(trainer, f"Unsloth: UNSLOTH_TOKEN_BALANCE ignored: {reason}.")+ return dataloader++ trainer.get_train_dataloader = get_train_dataloader++ if budget > 0:+ original_training_step = trainer.training_step+ trainer._unsloth_token_budget_splits = 0++ def training_step(+ model,+ inputs,+ num_items_in_batch = None,+ *args,+ **kwargs,+ ):+ accelerator = getattr(trainer, "accelerator", None)+ distributed_type = str(getattr(accelerator, "distributed_type", "NO")).upper()+ # Only None (loss would be a per-chunk mean), plain or DDP: DeepSpeed and FSDP count micro-steps themselves.+ if num_items_in_batch is None or not (+ distributed_type.endswith("NO") or distributed_type.endswith("MULTI_GPU")+ ):+ return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)+ reason = _exactness_blocker(trainer)+ if reason is not None:+ _note_once(trainer, f"Unsloth: UNSLOTH_TOKEN_BUDGET not splitting: {reason}.")+ return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)+ chunks = split_batch_by_token_budget(inputs, budget)+ if not _same_count_on_every_rank(len(chunks), accelerator):+ # Each chunk runs compute_loss (TRL gathers metrics there) and its own backward:+ # ranks with different counts would issue mismatched collectives and hang.+ _note_once(+ trainer,+ "Unsloth: UNSLOTH_TOKEN_BUDGET not splitting steps whose ranks need different chunk counts.",+ )+ return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)+ if len(chunks) == 1:+ return original_training_step(model, inputs, num_items_in_batch, *args, **kwargs)+ trainer._unsloth_token_budget_splits += 1+ total = None+ for i, chunk in enumerate(chunks):+ context = (+ accelerator.no_sync(model)+ if i < len(chunks) - 1 and accelerator is not None+ else contextlib.nullcontext()+ )+ with context:+ loss = original_training_step(model, chunk, num_items_in_batch, *args, **kwargs)+ total = loss if total is None else total + loss+ return total++ trainer.training_step = training_step