Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix returned logit scaling and softcapping when labels are provided (#13069)
cherry77-cloud9d0f0e1f484e
diff --git a/tests/test_fused_ce_logit_scale.py b/tests/test_fused_ce_logit_scale.pyindex fc00825171e..8db963d30ba 100644--- a/tests/test_fused_ce_logit_scale.py+++ b/tests/test_fused_ce_logit_scale.py@@ -349,6 +349,44 @@ def test_materialized_branch_is_handed_the_same_transforms(assert calls[0]["logit_softcapping"] == pytest.approx(softcapping)[email protected]("grad", [True, False], ids = ["grad", "no_grad"])+ "module,forward",+ [(llama_module, None), (mistral_module, MistralForCausalLM_fast_forward)],+ ids = ["llama", "mistral"],+)[email protected]("config,scale,softcapping", _SCALE_CASES, ids = _SCALE_IDS)+def test_labeled_call_returns_the_transformed_logits(+ config, scale, softcapping, module, forward, grad, monkeypatch+):+ """The kernel transforms the raw logits itself, so the returned ones are transformed+ separately, and only after the kernel has read them: without grad that is in place."""+ raw = F.linear(_hidden_states(), _lm_head_weight())+ expected = raw * scale+ if softcapping:+ expected = softcapping * torch.tanh(expected / softcapping)+ seen = []++ def spy(**kwargs):+ seen.append(kwargs["logits"].detach().clone())+ return torch.zeros((), requires_grad = True)++ monkeypatch.setattr(module, "fast_cross_entropy_loss", spy)+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")+ extra = {"input_ids": torch.zeros(BSZ, Q_LEN, dtype = torch.long)} if forward else {}+ with torch.set_grad_enabled(grad):+ output = (forward or CausalLM_fast_forward(None))(+ _FakeCausalLM(config()),+ labels = LABELS.clone(),+ return_dict = True,+ **extra,+ )++ assert torch.allclose(seen[0], raw, atol = 1e-5), "the kernel would transform these twice"+ assert torch.allclose(output.logits, expected, atol = 1e-5)+ assert output.logits.requires_grad == grad++def test_transforms_are_applied_scale_first_then_soft_cap():"""tanh does not commute with scaling, and no shipped family does both, so nothingabove separates the two orders."""diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 1df32c272dd..21ec7b524d2 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -1722,8 +1722,8 @@ def CausalLM_fast_forward(fast_forward_inference):logit_scaling = logit_scaling,n_items = n_items,)- else:- logits = apply_logit_transforms(logits, logit_softcapping, logit_scaling)+ # After the loss: the kernel reads the raw logits, and this is in place without grad.+ logits = apply_logit_transforms(logits, logit_softcapping, logit_scaling)if not return_dict:output = (logits,) + outputs[1:]diff --git a/unsloth/models/mistral.py b/unsloth/models/mistral.pyindex a80a34bdea7..e20471c851f 100644--- a/unsloth/models/mistral.py+++ b/unsloth/models/mistral.py@@ -407,9 +407,8 @@ def MistralForCausalLM_fast_forward(logit_scaling = logit_scaling,n_items = n_items,)- else:- # Inference returns the logits, so they carry the transforms themselves.- logits = apply_logit_transforms(logits, logit_softcapping, logit_scaling)+ # After the loss: the kernel reads the raw logits, and this is in place without grad.+ logits = apply_logit_transforms(logits, logit_softcapping, logit_scaling)if not return_dict:output = (logits,) + outputs[1:]