Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Bound the batched multi-LoRA loss memory: fused Triton cross entropy on GPU, chunked loss elsewhere (#16655)

Daniel Han480fd261e2fc
diff --git a/studio/backend/core/training/multi_lora_trainer.py b/studio/backend/core/training/multi_lora_trainer.py
index 3bcbfb1e60b..728ff2f6553 100644
--- a/studio/backend/core/training/multi_lora_trainer.py
+++ b/studio/backend/core/training/multi_lora_trainer.py
@@ -573,15 +573,12 @@ class _MemberLossScale:
self.good_steps = [0] * members
self.growth_interval = growth_interval
- def scale_backward(self, logits, adapter_ids):
- """GradScaler's loss scaling per token, in fp32: a logit's gradient comes only from its member's loss."""
+ def grad_scale(self, adapter_ids, device):
+ """GradScaler's loss scaling per row or token, in fp32: a logit's gradient comes only from its member's loss."""
import torch
- scale = torch.tensor(self.scale, dtype = torch.float32, device = logits.device)
- per_token = scale[adapter_ids.to(logits.device).clamp_min(0)]
- per_token = per_token[:, None, None] if per_token.dim() == 1 else per_token[..., None]
- logits.register_hook(lambda grad: grad * per_token)
- return logits
+ scale = torch.tensor(self.scale, dtype = torch.float32, device = device)
+ return scale[adapter_ids.to(device).clamp_min(0)]
def unscale_(self, slot: int, params: list) -> bool:
"""Unscales the member's gradients; False (and its gradients dropped) when any is not finite."""
@@ -1363,8 +1360,6 @@ def _build_sft_trainer(
attention_mask = inputs["attention_mask"],
use_cache = False,
).logits
- if self.loss_scale is not None:
- logits = self.loss_scale.scale_backward(logits.float(), ids)
loss, per = per_adapter_loss(
logits,
inputs["labels"],
@@ -1372,6 +1367,9 @@ def _build_sft_trainer(
U,
model = self.model,
num_items_in_batch = num_items_in_batch,
+ grad_scale = None
+ if self.loss_scale is None
+ else self.loss_scale.grad_scale(ids, logits.device),
)
self.round_losses += per.detach().float().cpu()
return loss
diff --git a/tests/test_multi_lora.py b/tests/test_multi_lora.py
index b359f5b1c72..67480ff93ac 100644
--- a/tests/test_multi_lora.py
+++ b/tests/test_multi_lora.py
@@ -171,6 +171,226 @@ def test_a_single_string_target_is_one_name():
assert ml.inject_multi_lora(model, "1", 2) == ["1"]
+def _dense_per_adapter_loss(
+ logits,
+ labels,
+ ids,
+ num_adapters,
+ ignore_index = -100,
+):
+ shift = logits[:, :-1].float()
+ token = torch.nn.functional.cross_entropy(
+ shift.reshape(-1, shift.shape[-1]),
+ labels[:, 1:].reshape(-1),
+ ignore_index = ignore_index,
+ reduction = "none",
+ ).view(labels[:, 1:].shape)
+ valid = (labels[:, 1:] != ignore_index).float()
+ ids = ml._label_adapter_ids(ids, labels)
+ keep = (ids >= 0).float()
+ ids = ids.clamp_min(0).reshape(-1)
+ sums = torch.zeros(num_adapters, device = logits.device).index_add(
+ 0, ids, (token * keep).reshape(-1)
+ )
+ counts = torch.zeros(num_adapters, device = logits.device).index_add(
+ 0, ids, (valid * keep).reshape(-1)
+ )
+ losses = sums / counts.clamp_min(1)
+ return losses.sum(), losses.detach()
+
+
+# Losses agree to fp32 rounding; gradients to rounding in fp32, one rounding into the logits dtype in bf16.
+_CHUNKED_TOL = {
+ torch.float32: dict(rtol = 1e-6, atol = 1e-7),
+ torch.bfloat16: dict(rtol = 2**-7, atol = 1e-6),
+}
+
+
+def _chunked_vs_dense(
+ dtype,
+ per_token_ids,
+ V = 7,
+):
+ g = torch.Generator().manual_seed(0)
+ B, T, U = 4, 9, 3
+ logits = (torch.randn(B, T, V, generator = g) * 3).to(dtype)
+ labels = torch.randint(0, V, (B, T), generator = g)
+ labels[1, 4:] = -100
+ labels[3, :] = -100
+ if per_token_ids:
+ ids = torch.randint(-1, U, (B, T), generator = g)
+ else:
+ ids = torch.tensor([2, 0, 2, 1])
+
+ def run(fn):
+ x = logits.clone().requires_grad_()
+ total, losses = fn(x, labels, ids, U)
+ (total * 1.5).backward()
+ return total.detach(), losses, x.grad
+
+ return run(ml.per_adapter_loss), run(_dense_per_adapter_loss)
+
+
+def _assert_chunked_matches(new, old, dtype):
+ for a, b in zip(new, old):
+ assert a.dtype == b.dtype
+ torch.testing.assert_close(a, b, **_CHUNKED_TOL[dtype])
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
[email protected]("chunk_elements", [1, 7 * 3, 7 * 8 * 2, 1 << 25])
[email protected]("per_token_ids", [False, True])
+def test_chunked_per_adapter_loss_matches_the_dense_loss(
+ monkeypatch, dtype, chunk_elements, per_token_ids
+):
+ monkeypatch.setattr(ml, "_CE_CHUNK_ELEMENTS", chunk_elements)
+ new, old = _chunked_vs_dense(dtype, per_token_ids)
+ _assert_chunked_matches(new, old, dtype)
+ err = (new[2].float() - old[2].float()).abs().max().item()
+ print(f"{dtype} chunk={chunk_elements} max |grad diff| {err:.3g}")
+
+
+ "mutation",
+ [
+ # no onehot: the label's logit is never pushed up
+ ("-torch.ones_like(g)[:, None]", "0 * torch.ones_like(g)[:, None]"),
+ # no logsumexp: exp(x) instead of softmax
+ (".sub_(lse[b, t:e, None])", ""),
+ ],
+)
[email protected]("dtype", [torch.float32, torch.bfloat16])
+def test_the_chunked_parity_check_catches_a_broken_backward(monkeypatch, mutation, dtype):
+ import inspect
+
+ src = textwrap.dedent(inspect.getsource(ml._ChunkedTokenCrossEntropy))
+ assert mutation[0] in src
+ namespace = dict(vars(ml))
+ exec(src.replace(mutation[0], mutation[1], 1), namespace)
+ monkeypatch.setattr(ml, "_ChunkedTokenCrossEntropy", namespace["_ChunkedTokenCrossEntropy"])
+ new, old = _chunked_vs_dense(dtype, per_token_ids = True)
+ with pytest.raises(AssertionError):
+ _assert_chunked_matches(new, old, dtype)
+
+
+def test_chunked_per_adapter_loss_saves_no_full_vocab_fp32_copy(monkeypatch):
+ B, T, V, U = 8, 64, 4096, 4
+ logits = torch.randn(B, T, V).to(torch.bfloat16).requires_grad_()
+ labels = torch.randint(0, V, (B, T))
+ ids = torch.arange(B) % U
+ full = B * (T - 1) * V
+
+ def saved_bytes(fn):
+ saved = []
+
+ def pack(t):
+ saved.append(t)
+ return t
+
+ with torch.autograd.graph.saved_tensors_hooks(pack, lambda t: t):
+ fn(logits, labels, ids, U)
+ own = logits.untyped_storage().data_ptr()
+ return sum(
+ t.numel() * t.element_size() for t in saved if t.untyped_storage().data_ptr() != own
+ )
+
+ dense = saved_bytes(_dense_per_adapter_loss)
+ chunked = saved_bytes(ml.per_adapter_loss)
+ assert dense >= 8 * full # the fp32 copy and cross_entropy's fp32 log-softmax
+ assert chunked < full // 100, (chunked, dense)
+
+
[email protected]("per_token_ids", [False, True])
+def test_grad_scale_matches_scaling_an_fp32_copy_of_the_logits(per_token_ids):
+ # The fp16 loss-scale path used to scale the gradient of a full fp32 copy of the logits.
+ g = torch.Generator().manual_seed(1)
+ B, T, V, U = 3, 6, 11, 2
+ logits = (torch.randn(B, T, V, generator = g) * 3).half()
+ labels = torch.randint(0, V, (B, T), generator = g)
+ labels[0, 3:] = -100
+ ids = torch.randint(-1, U, (B, T), generator = g) if per_token_ids else torch.tensor([1, 0, 1])
+ scales = torch.tensor([2.0**16, 2.0**10])
+ per = scales[ids.clamp_min(0)]
+
+ x = logits.clone().requires_grad_()
+ up = x.float()
+ up.register_hook(lambda grad: grad * (per[:, None, None] if per.dim() == 1 else per[..., None]))
+ old_total, old_losses = ml.per_adapter_loss(up, labels, ids, U)
+ old_total.backward()
+
+ y = logits.clone().requires_grad_()
+ total, losses = ml.per_adapter_loss(y, labels, ids, U, grad_scale = per)
+ total.backward()
+ assert torch.equal(total, old_total) and torch.equal(losses, old_losses)
+ assert torch.equal(y.grad, x.grad)
+
+
+def test_chunked_per_adapter_loss_handles_one_token_rows():
+ logits = torch.randn(2, 1, 5, requires_grad = True)
+ total, losses = ml.per_adapter_loss(
+ logits, torch.zeros(2, 1, dtype = torch.long), torch.tensor([0, 1]), 2
+ )
+ total.backward()
+ assert total.item() == 0 and torch.equal(losses, torch.zeros(2))
+ assert torch.equal(logits.grad, torch.zeros_like(logits))
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA GPU")
[email protected]("dtype", [torch.float32, torch.bfloat16, torch.float16])
[email protected]("V", [7, 70000])
+def test_gpu_per_adapter_loss_matches_the_dense_loss(dtype, V):
+ if dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():
+ pytest.skip("no bf16")
+ g = torch.Generator().manual_seed(0)
+ B, T, U = 4, 9, 3
+ logits = (torch.randn(B, T, V, generator = g) * 3).to(dtype).cuda()
+ labels = torch.randint(0, V, (B, T), generator = g).cuda()
+ labels[1, 4:] = -100
+ ids = torch.randint(-1, U, (B, T), generator = g).cuda()
+ scale = torch.tensor([2.0**10, 1.0, 2.0**4], device = "cuda")[ids.clamp_min(0)]
+
+ def run(fn, **kw):
+ x = logits.clone().requires_grad_()
+ total, losses = fn(x, labels, ids, U, **kw)
+ total.backward()
+ return total.detach(), losses, x.grad
+
+ new = run(ml.per_adapter_loss, grad_scale = scale)
+ # Reference: dense fp32 loss, scaled in fp32, rounded once into the logits dtype.
+ x = logits.float().requires_grad_()
+ x.register_hook(lambda grad: grad * scale[..., None])
+ total, losses = _dense_per_adapter_loss(x, labels, ids, U)
+ total.backward()
+ rtol = {torch.float32: 1e-5, torch.bfloat16: 2**-7, torch.float16: 2**-10}[dtype]
+ torch.testing.assert_close(new[0], total.detach(), rtol = 1e-5, atol = 1e-6)
+ torch.testing.assert_close(new[1], losses, rtol = 1e-5, atol = 1e-6)
+ torch.testing.assert_close(new[2], x.grad.to(dtype), rtol = rtol, atol = 1e-6)
+
+
+def test_the_loss_follows_the_triton_switch(monkeypatch):
+ # Off = the PyTorch loss, as for the layers; on CUDA / XPU = the fused Triton cross entropy.
+ import unsloth.kernels.cross_entropy_loss as ce
+
+ calls = []
+ monkeypatch.setattr(ce.Fast_CrossEntropyLoss, "apply", lambda *a: calls.append(1) or 1 / 0)
+ monkeypatch.setattr(ml, "_use_triton", lambda x: ml._TRITON_ENABLED)
+ logits = torch.randn(2, 4, 5)
+ labels = torch.randint(0, 5, (2, 4))
+ ml.set_multi_lora_triton(False)
+ try:
+ ml.per_adapter_loss(logits, labels, torch.tensor([0, 1]), 2)
+ assert calls == []
+ ml.set_multi_lora_triton(True)
+ if ml._TRITON_ENABLED:
+ with pytest.raises(ZeroDivisionError):
+ ml.per_adapter_loss(logits, labels, torch.tensor([0, 1]), 2)
+ assert calls == [1]
+ finally:
+ ml.set_multi_lora_triton(
+ ml.HAS_TRITON and os.environ.get("UNSLOTH_MULTI_LORA_TRITON", "1") != "0"
+ )
+
+
def test_per_adapter_loss_is_each_adapters_own_mean():
logits = torch.randn(3, 5, 7)
labels = torch.randint(0, 7, (3, 5))
diff --git a/unsloth/kernels/multi_lora.py b/unsloth/kernels/multi_lora.py
index 856c9291daa..fa2e46c6eb8 100644
--- a/unsloth/kernels/multi_lora.py
+++ b/unsloth/kernels/multi_lora.py
@@ -1560,6 +1560,69 @@ def adapter_token_counts(
)
+# fp32 logits per chunk: a full fp32 copy of B x T logits is several times the bf16 logits and OOMs.
+_CE_CHUNK_ELEMENTS = 1 << 25
+
+
+class _ChunkedTokenCrossEntropy(torch.autograd.Function):
+ """Chunked per-token cross entropy of ``logits[:, :-1]``; saves only logits, labels and per-token fp32
+ max and log-sum-exp (no fp32 logits copy) and backward forms ``softmax - onehot`` from them."""
+
+ @staticmethod
+ def forward(ctx, logits, labels, ignore_index, chunk):
+ B, T1 = labels.shape
+ loss = torch.empty((B, T1), dtype = torch.float32, device = logits.device)
+ # Max and log-sum-exp kept apart, as log_softmax rounds them: x - (max + lse) loses ulp(max).
+ mx = torch.empty((B, T1), dtype = torch.float32, device = logits.device)
+ lse = torch.empty((B, T1), dtype = torch.float32, device = logits.device)
+ for b in range(B):
+ for t in range(0, T1, chunk):
+ e = min(t + chunk, T1)
+ y = labels[b, t:e]
+ valid = y != ignore_index
+ z = logits[b, t:e].float()
+ m = z.amax(1, keepdim = True)
+ z = z - m
+ s = z.exp().sum(1).log()
+ picked = z.gather(1, torch.where(valid, y, 0)[:, None]).squeeze(1)
+ loss[b, t:e] = torch.where(valid, s - picked, 0.0)
+ mx[b, t:e] = m.squeeze(1)
+ # +inf on ignored tokens makes their softmax 0.
+ lse[b, t:e] = torch.where(valid, s, float("inf"))
+ ctx.save_for_backward(logits, labels, mx, lse)
+ ctx.ignore_index, ctx.chunk = ignore_index, chunk
+ return loss
+
+ @staticmethod
+ def backward(ctx, grad_loss):
+ logits, labels, mx, lse = ctx.saved_tensors
+ grad = torch.zeros_like(logits)
+ B, T1 = labels.shape
+ for b in range(B):
+ for t in range(0, T1, ctx.chunk):
+ e = min(t + ctx.chunk, T1)
+ y = labels[b, t:e]
+ valid = y != ctx.ignore_index
+ g = torch.where(valid, grad_loss[b, t:e].float(), 0.0)
+ p = (logits[b, t:e].float() - mx[b, t:e, None]).sub_(lse[b, t:e, None]).exp_()
+ p.scatter_add_(1, torch.where(valid, y, 0)[:, None], -torch.ones_like(g)[:, None])
+ grad[b, t:e] = p.mul_(g[:, None])
+ return grad, None, None, None
+
+
+def _token_cross_entropy(logits, shift_labels, ignore_index):
+ if ignore_index == -100 and _use_triton(logits):
+ from .cross_entropy_loss import Fast_CrossEntropyLoss
+
+ # Full rows, no logits[:, :-1] copy: the last position's -100 label gives it zero loss and gradient.
+ B, T, V = logits.shape
+ labels = F.pad(shift_labels, (0, 1), value = -100)
+ loss = Fast_CrossEntropyLoss.apply(logits.reshape(B * T, V), labels.reshape(-1), 0, 0)
+ return loss.view(B, T)[:, :-1]
+ chunk = max(1, _CE_CHUNK_ELEMENTS // max(1, logits.shape[-1]))
+ return _ChunkedTokenCrossEntropy.apply(logits, shift_labels, ignore_index, chunk)
+
+
def per_adapter_loss(
logits: torch.Tensor,
labels: torch.Tensor,
@@ -1569,21 +1632,23 @@ def per_adapter_loss(
process_group = None,
model: Optional[nn.Module] = None,
num_items_in_batch: Optional[torch.Tensor] = None,
+ grad_scale: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Causal LM loss averaged within each adapter, summed over adapters.
``row_adapter_ids``: (B,) or per token (B, T). Pass ``model`` for ``drop_absent_adapter_grads_``, and under
gradient accumulation ``num_items_in_batch`` = per-adapter ``adapter_token_counts`` summed over the window.
+ ``grad_scale``: per-row (B,) or per-token (B, T) fp32 factor on each logit's gradient (fp16 loss
+ scaling), applied before the gradient is rounded to the logits dtype; values stay unscaled.
Returns (total, this rank's per-adapter losses).
"""
- shift_logits = logits[:, :-1].float()
shift_labels = labels[:, 1:]
- token_loss = F.cross_entropy(
- shift_logits.reshape(-1, shift_logits.shape[-1]),
- shift_labels.reshape(-1),
- ignore_index = ignore_index,
- reduction = "none",
- ).view(shift_labels.shape)
+ token_loss = _token_cross_entropy(logits, shift_labels.to(logits.device), ignore_index)
+ if grad_scale is not None:
+ scale = grad_scale.to(token_loss)
+ scale = scale[:, :-1] if scale.dim() == 2 else scale[:, None]
+ # Same value, gradient times scale.
+ token_loss = token_loss.detach() + (token_loss - token_loss.detach()) * scale
valid = (shift_labels != ignore_index).float()
ids = _label_adapter_ids(row_adapter_ids, labels)
# Tokens with no adapter add zero to slot 0 instead of being masked out: no host sync under compile.