Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix returned logit scaling and softcapping when labels are provided (#13069)

cherry77-cloud9d0f0e1f484e
diff --git a/tests/test_fused_ce_logit_scale.py b/tests/test_fused_ce_logit_scale.py
index fc00825171e..8db963d30ba 100644
--- a/tests/test_fused_ce_logit_scale.py
+++ b/tests/test_fused_ce_logit_scale.py
@@ -349,6 +349,44 @@ def test_materialized_branch_is_handed_the_same_transforms(
assert calls[0]["logit_softcapping"] == pytest.approx(softcapping)
[email protected]("grad", [True, False], ids = ["grad", "no_grad"])
+ "module,forward",
+ [(llama_module, None), (mistral_module, MistralForCausalLM_fast_forward)],
+ ids = ["llama", "mistral"],
+)
[email protected]("config,scale,softcapping", _SCALE_CASES, ids = _SCALE_IDS)
+def test_labeled_call_returns_the_transformed_logits(
+ config, scale, softcapping, module, forward, grad, monkeypatch
+):
+ """The kernel transforms the raw logits itself, so the returned ones are transformed
+ separately, and only after the kernel has read them: without grad that is in place."""
+ raw = F.linear(_hidden_states(), _lm_head_weight())
+ expected = raw * scale
+ if softcapping:
+ expected = softcapping * torch.tanh(expected / softcapping)
+ seen = []
+
+ def spy(**kwargs):
+ seen.append(kwargs["logits"].detach().clone())
+ return torch.zeros((), requires_grad = True)
+
+ monkeypatch.setattr(module, "fast_cross_entropy_loss", spy)
+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")
+ extra = {"input_ids": torch.zeros(BSZ, Q_LEN, dtype = torch.long)} if forward else {}
+ with torch.set_grad_enabled(grad):
+ output = (forward or CausalLM_fast_forward(None))(
+ _FakeCausalLM(config()),
+ labels = LABELS.clone(),
+ return_dict = True,
+ **extra,
+ )
+
+ assert torch.allclose(seen[0], raw, atol = 1e-5), "the kernel would transform these twice"
+ assert torch.allclose(output.logits, expected, atol = 1e-5)
+ assert output.logits.requires_grad == grad
+
+
def test_transforms_are_applied_scale_first_then_soft_cap():
"""tanh does not commute with scaling, and no shipped family does both, so nothing
above separates the two orders."""
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 1df32c272dd..21ec7b524d2 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -1722,8 +1722,8 @@ def CausalLM_fast_forward(fast_forward_inference):
logit_scaling = logit_scaling,
n_items = n_items,
)
- else:
- logits = apply_logit_transforms(logits, logit_softcapping, logit_scaling)
+ # After the loss: the kernel reads the raw logits, and this is in place without grad.
+ logits = apply_logit_transforms(logits, logit_softcapping, logit_scaling)
if not return_dict:
output = (logits,) + outputs[1:]
diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.py
index a80a34bdea7..e20471c851f 100644
--- a/unsloth/models/mistral.py
+++ b/unsloth/models/mistral.py
@@ -407,9 +407,8 @@ def MistralForCausalLM_fast_forward(
logit_scaling = logit_scaling,
n_items = n_items,
)
- else:
- # Inference returns the logits, so they carry the transforms themselves.
- logits = apply_logit_transforms(logits, logit_softcapping, logit_scaling)
+ # After the loss: the kernel reads the raw logits, and this is in place without grad.
+ logits = apply_logit_transforms(logits, logit_softcapping, logit_scaling)
if not return_dict:
output = (logits,) + outputs[1:]