Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Bound the batched multi-LoRA loss memory: fused Triton cross entropy on GPU, chunked loss elsewhere (#16655)
Daniel Han480fd261e2fc
diff --git a/studio/backend/core/training/multi_lora_trainer.py b/studio/backend/core/training/multi_lora_trainer.pyindex 3bcbfb1e60b..728ff2f6553 100644--- a/studio/backend/core/training/multi_lora_trainer.py+++ b/studio/backend/core/training/multi_lora_trainer.py@@ -573,15 +573,12 @@ class _MemberLossScale:self.good_steps = [0] * membersself.growth_interval = growth_interval- def scale_backward(self, logits, adapter_ids):- """GradScaler's loss scaling per token, in fp32: a logit's gradient comes only from its member's loss."""+ def grad_scale(self, adapter_ids, device):+ """GradScaler's loss scaling per row or token, in fp32: a logit's gradient comes only from its member's loss."""import torch- scale = torch.tensor(self.scale, dtype = torch.float32, device = logits.device)- per_token = scale[adapter_ids.to(logits.device).clamp_min(0)]- per_token = per_token[:, None, None] if per_token.dim() == 1 else per_token[..., None]- logits.register_hook(lambda grad: grad * per_token)- return logits+ scale = torch.tensor(self.scale, dtype = torch.float32, device = device)+ return scale[adapter_ids.to(device).clamp_min(0)]def unscale_(self, slot: int, params: list) -> bool:"""Unscales the member's gradients; False (and its gradients dropped) when any is not finite."""@@ -1363,8 +1360,6 @@ def _build_sft_trainer(attention_mask = inputs["attention_mask"],use_cache = False,).logits- if self.loss_scale is not None:- logits = self.loss_scale.scale_backward(logits.float(), ids)loss, per = per_adapter_loss(logits,inputs["labels"],@@ -1372,6 +1367,9 @@ def _build_sft_trainer(U,model = self.model,num_items_in_batch = num_items_in_batch,+ grad_scale = None+ if self.loss_scale is None+ else self.loss_scale.grad_scale(ids, logits.device),)self.round_losses += per.detach().float().cpu()return lossdiff --git a/tests/test_multi_lora.py b/tests/test_multi_lora.pyindex b359f5b1c72..67480ff93ac 100644--- a/tests/test_multi_lora.py+++ b/tests/test_multi_lora.py@@ -171,6 +171,226 @@ def test_a_single_string_target_is_one_name():assert ml.inject_multi_lora(model, "1", 2) == ["1"]+def _dense_per_adapter_loss(+ logits,+ labels,+ ids,+ num_adapters,+ ignore_index = -100,+):+ shift = logits[:, :-1].float()+ token = torch.nn.functional.cross_entropy(+ shift.reshape(-1, shift.shape[-1]),+ labels[:, 1:].reshape(-1),+ ignore_index = ignore_index,+ reduction = "none",+ ).view(labels[:, 1:].shape)+ valid = (labels[:, 1:] != ignore_index).float()+ ids = ml._label_adapter_ids(ids, labels)+ keep = (ids >= 0).float()+ ids = ids.clamp_min(0).reshape(-1)+ sums = torch.zeros(num_adapters, device = logits.device).index_add(+ 0, ids, (token * keep).reshape(-1)+ )+ counts = torch.zeros(num_adapters, device = logits.device).index_add(+ 0, ids, (valid * keep).reshape(-1)+ )+ losses = sums / counts.clamp_min(1)+ return losses.sum(), losses.detach()+++# Losses agree to fp32 rounding; gradients to rounding in fp32, one rounding into the logits dtype in bf16.+_CHUNKED_TOL = {+ torch.float32: dict(rtol = 1e-6, atol = 1e-7),+ torch.bfloat16: dict(rtol = 2**-7, atol = 1e-6),+}+++def _chunked_vs_dense(+ dtype,+ per_token_ids,+ V = 7,+):+ g = torch.Generator().manual_seed(0)+ B, T, U = 4, 9, 3+ logits = (torch.randn(B, T, V, generator = g) * 3).to(dtype)+ labels = torch.randint(0, V, (B, T), generator = g)+ labels[1, 4:] = -100+ labels[3, :] = -100+ if per_token_ids:+ ids = torch.randint(-1, U, (B, T), generator = g)+ else:+ ids = torch.tensor([2, 0, 2, 1])++ def run(fn):+ x = logits.clone().requires_grad_()+ total, losses = fn(x, labels, ids, U)+ (total * 1.5).backward()+ return total.detach(), losses, x.grad++ return run(ml.per_adapter_loss), run(_dense_per_adapter_loss)+++def _assert_chunked_matches(new, old, dtype):+ for a, b in zip(new, old):+ assert a.dtype == b.dtype+ torch.testing.assert_close(a, b, **_CHUNKED_TOL[dtype])++[email protected]("dtype", [torch.float32, torch.bfloat16])[email protected]("chunk_elements", [1, 7 * 3, 7 * 8 * 2, 1 << 25])[email protected]("per_token_ids", [False, True])+def test_chunked_per_adapter_loss_matches_the_dense_loss(+ monkeypatch, dtype, chunk_elements, per_token_ids+):+ monkeypatch.setattr(ml, "_CE_CHUNK_ELEMENTS", chunk_elements)+ new, old = _chunked_vs_dense(dtype, per_token_ids)+ _assert_chunked_matches(new, old, dtype)+ err = (new[2].float() - old[2].float()).abs().max().item()+ print(f"{dtype} chunk={chunk_elements} max |grad diff| {err:.3g}")+++ "mutation",+ [+ # no onehot: the label's logit is never pushed up+ ("-torch.ones_like(g)[:, None]", "0 * torch.ones_like(g)[:, None]"),+ # no logsumexp: exp(x) instead of softmax+ (".sub_(lse[b, t:e, None])", ""),+ ],+)[email protected]("dtype", [torch.float32, torch.bfloat16])+def test_the_chunked_parity_check_catches_a_broken_backward(monkeypatch, mutation, dtype):+ import inspect++ src = textwrap.dedent(inspect.getsource(ml._ChunkedTokenCrossEntropy))+ assert mutation[0] in src+ namespace = dict(vars(ml))+ exec(src.replace(mutation[0], mutation[1], 1), namespace)+ monkeypatch.setattr(ml, "_ChunkedTokenCrossEntropy", namespace["_ChunkedTokenCrossEntropy"])+ new, old = _chunked_vs_dense(dtype, per_token_ids = True)+ with pytest.raises(AssertionError):+ _assert_chunked_matches(new, old, dtype)+++def test_chunked_per_adapter_loss_saves_no_full_vocab_fp32_copy(monkeypatch):+ B, T, V, U = 8, 64, 4096, 4+ logits = torch.randn(B, T, V).to(torch.bfloat16).requires_grad_()+ labels = torch.randint(0, V, (B, T))+ ids = torch.arange(B) % U+ full = B * (T - 1) * V++ def saved_bytes(fn):+ saved = []++ def pack(t):+ saved.append(t)+ return t++ with torch.autograd.graph.saved_tensors_hooks(pack, lambda t: t):+ fn(logits, labels, ids, U)+ own = logits.untyped_storage().data_ptr()+ return sum(+ t.numel() * t.element_size() for t in saved if t.untyped_storage().data_ptr() != own+ )++ dense = saved_bytes(_dense_per_adapter_loss)+ chunked = saved_bytes(ml.per_adapter_loss)+ assert dense >= 8 * full # the fp32 copy and cross_entropy's fp32 log-softmax+ assert chunked < full // 100, (chunked, dense)++[email protected]("per_token_ids", [False, True])+def test_grad_scale_matches_scaling_an_fp32_copy_of_the_logits(per_token_ids):+ # The fp16 loss-scale path used to scale the gradient of a full fp32 copy of the logits.+ g = torch.Generator().manual_seed(1)+ B, T, V, U = 3, 6, 11, 2+ logits = (torch.randn(B, T, V, generator = g) * 3).half()+ labels = torch.randint(0, V, (B, T), generator = g)+ labels[0, 3:] = -100+ ids = torch.randint(-1, U, (B, T), generator = g) if per_token_ids else torch.tensor([1, 0, 1])+ scales = torch.tensor([2.0**16, 2.0**10])+ per = scales[ids.clamp_min(0)]++ x = logits.clone().requires_grad_()+ up = x.float()+ up.register_hook(lambda grad: grad * (per[:, None, None] if per.dim() == 1 else per[..., None]))+ old_total, old_losses = ml.per_adapter_loss(up, labels, ids, U)+ old_total.backward()++ y = logits.clone().requires_grad_()+ total, losses = ml.per_adapter_loss(y, labels, ids, U, grad_scale = per)+ total.backward()+ assert torch.equal(total, old_total) and torch.equal(losses, old_losses)+ assert torch.equal(y.grad, x.grad)+++def test_chunked_per_adapter_loss_handles_one_token_rows():+ logits = torch.randn(2, 1, 5, requires_grad = True)+ total, losses = ml.per_adapter_loss(+ logits, torch.zeros(2, 1, dtype = torch.long), torch.tensor([0, 1]), 2+ )+ total.backward()+ assert total.item() == 0 and torch.equal(losses, torch.zeros(2))+ assert torch.equal(logits.grad, torch.zeros_like(logits))++[email protected](not has_real_cuda(), reason = "needs a CUDA GPU")[email protected]("dtype", [torch.float32, torch.bfloat16, torch.float16])[email protected]("V", [7, 70000])+def test_gpu_per_adapter_loss_matches_the_dense_loss(dtype, V):+ if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():+ pytest.skip("no bf16")+ g = torch.Generator().manual_seed(0)+ B, T, U = 4, 9, 3+ logits = (torch.randn(B, T, V, generator = g) * 3).to(dtype).cuda()+ labels = torch.randint(0, V, (B, T), generator = g).cuda()+ labels[1, 4:] = -100+ ids = torch.randint(-1, U, (B, T), generator = g).cuda()+ scale = torch.tensor([2.0**10, 1.0, 2.0**4], device = "cuda")[ids.clamp_min(0)]++ def run(fn, **kw):+ x = logits.clone().requires_grad_()+ total, losses = fn(x, labels, ids, U, **kw)+ total.backward()+ return total.detach(), losses, x.grad++ new = run(ml.per_adapter_loss, grad_scale = scale)+ # Reference: dense fp32 loss, scaled in fp32, rounded once into the logits dtype.+ x = logits.float().requires_grad_()+ x.register_hook(lambda grad: grad * scale[..., None])+ total, losses = _dense_per_adapter_loss(x, labels, ids, U)+ total.backward()+ rtol = {torch.float32: 1e-5, torch.bfloat16: 2**-7, torch.float16: 2**-10}[dtype]+ torch.testing.assert_close(new[0], total.detach(), rtol = 1e-5, atol = 1e-6)+ torch.testing.assert_close(new[1], losses, rtol = 1e-5, atol = 1e-6)+ torch.testing.assert_close(new[2], x.grad.to(dtype), rtol = rtol, atol = 1e-6)+++def test_the_loss_follows_the_triton_switch(monkeypatch):+ # Off = the PyTorch loss, as for the layers; on CUDA / XPU = the fused Triton cross entropy.+ import unsloth.kernels.cross_entropy_loss as ce++ calls = []+ monkeypatch.setattr(ce.Fast_CrossEntropyLoss, "apply", lambda *a: calls.append(1) or 1 / 0)+ monkeypatch.setattr(ml, "_use_triton", lambda x: ml._TRITON_ENABLED)+ logits = torch.randn(2, 4, 5)+ labels = torch.randint(0, 5, (2, 4))+ ml.set_multi_lora_triton(False)+ try:+ ml.per_adapter_loss(logits, labels, torch.tensor([0, 1]), 2)+ assert calls == []+ ml.set_multi_lora_triton(True)+ if ml._TRITON_ENABLED:+ with pytest.raises(ZeroDivisionError):+ ml.per_adapter_loss(logits, labels, torch.tensor([0, 1]), 2)+ assert calls == [1]+ finally:+ ml.set_multi_lora_triton(+ ml.HAS_TRITON and os.environ.get("UNSLOTH_MULTI_LORA_TRITON", "1") != "0"+ )++def test_per_adapter_loss_is_each_adapters_own_mean():logits = torch.randn(3, 5, 7)labels = torch.randint(0, 7, (3, 5))diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.pyindex 856c9291daa..fa2e46c6eb8 100644--- a/unsloth/kernels/multi_lora.py+++ b/unsloth/kernels/multi_lora.py@@ -1560,6 +1560,69 @@ def adapter_token_counts()+# fp32 logits per chunk: a full fp32 copy of B x T logits is several times the bf16 logits and OOMs.+_CE_CHUNK_ELEMENTS = 1 << 25+++class _ChunkedTokenCrossEntropy(torch.autograd.Function):+ """Chunked per-token cross entropy of ``logits[:, :-1]``; saves only logits, labels and per-token fp32+ max and log-sum-exp (no fp32 logits copy) and backward forms ``softmax - onehot`` from them."""++ @staticmethod+ def forward(ctx, logits, labels, ignore_index, chunk):+ B, T1 = labels.shape+ loss = torch.empty((B, T1), dtype = torch.float32, device = logits.device)+ # Max and log-sum-exp kept apart, as log_softmax rounds them: x - (max + lse) loses ulp(max).+ mx = torch.empty((B, T1), dtype = torch.float32, device = logits.device)+ lse = torch.empty((B, T1), dtype = torch.float32, device = logits.device)+ for b in range(B):+ for t in range(0, T1, chunk):+ e = min(t + chunk, T1)+ y = labels[b, t:e]+ valid = y != ignore_index+ z = logits[b, t:e].float()+ m = z.amax(1, keepdim = True)+ z = z - m+ s = z.exp().sum(1).log()+ picked = z.gather(1, torch.where(valid, y, 0)[:, None]).squeeze(1)+ loss[b, t:e] = torch.where(valid, s - picked, 0.0)+ mx[b, t:e] = m.squeeze(1)+ # +inf on ignored tokens makes their softmax 0.+ lse[b, t:e] = torch.where(valid, s, float("inf"))+ ctx.save_for_backward(logits, labels, mx, lse)+ ctx.ignore_index, ctx.chunk = ignore_index, chunk+ return loss++ @staticmethod+ def backward(ctx, grad_loss):+ logits, labels, mx, lse = ctx.saved_tensors+ grad = torch.zeros_like(logits)+ B, T1 = labels.shape+ for b in range(B):+ for t in range(0, T1, ctx.chunk):+ e = min(t + ctx.chunk, T1)+ y = labels[b, t:e]+ valid = y != ctx.ignore_index+ g = torch.where(valid, grad_loss[b, t:e].float(), 0.0)+ p = (logits[b, t:e].float() - mx[b, t:e, None]).sub_(lse[b, t:e, None]).exp_()+ p.scatter_add_(1, torch.where(valid, y, 0)[:, None], -torch.ones_like(g)[:, None])+ grad[b, t:e] = p.mul_(g[:, None])+ return grad, None, None, None+++def _token_cross_entropy(logits, shift_labels, ignore_index):+ if ignore_index == -100 and _use_triton(logits):+ from .cross_entropy_loss import Fast_CrossEntropyLoss++ # Full rows, no logits[:, :-1] copy: the last position's -100 label gives it zero loss and gradient.+ B, T, V = logits.shape+ labels = F.pad(shift_labels, (0, 1), value = -100)+ loss = Fast_CrossEntropyLoss.apply(logits.reshape(B * T, V), labels.reshape(-1), 0, 0)+ return loss.view(B, T)[:, :-1]+ chunk = max(1, _CE_CHUNK_ELEMENTS // max(1, logits.shape[-1]))+ return _ChunkedTokenCrossEntropy.apply(logits, shift_labels, ignore_index, chunk)++def per_adapter_loss(logits: torch.Tensor,labels: torch.Tensor,@@ -1569,21 +1632,23 @@ def per_adapter_loss(process_group = None,model: Optional[nn.Module] = None,num_items_in_batch: Optional[torch.Tensor] = None,+ grad_scale: Optional[torch.Tensor] = None,) -> tuple[torch.Tensor, torch.Tensor]:"""Causal LM loss averaged within each adapter, summed over adapters.``row_adapter_ids``: (B,) or per token (B, T). Pass ``model`` for ``drop_absent_adapter_grads_``, and undergradient accumulation ``num_items_in_batch`` = per-adapter ``adapter_token_counts`` summed over the window.+ ``grad_scale``: per-row (B,) or per-token (B, T) fp32 factor on each logit's gradient (fp16 loss+ scaling), applied before the gradient is rounded to the logits dtype; values stay unscaled.Returns (total, this rank's per-adapter losses)."""- shift_logits = logits[:, :-1].float()shift_labels = labels[:, 1:]- token_loss = F.cross_entropy(- shift_logits.reshape(-1, shift_logits.shape[-1]),- shift_labels.reshape(-1),- ignore_index = ignore_index,- reduction = "none",- ).view(shift_labels.shape)+ token_loss = _token_cross_entropy(logits, shift_labels.to(logits.device), ignore_index)+ if grad_scale is not None:+ scale = grad_scale.to(token_loss)+ scale = scale[:, :-1] if scale.dim() == 2 else scale[:, None]+ # Same value, gradient times scale.+ token_loss = token_loss.detach() + (token_loss - token_loss.detach()) * scalevalid = (shift_labels != ignore_index).float()ids = _label_adapter_ids(row_adapter_ids, labels)# Tokens with no adapter add zero to slot 0 instead of being masked out: no host sync under compile.