Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Each fused loss must apply its model's logit scale: Cohere multiplies, Granite divides."""
from __future__ import annotations
from types import SimpleNamespace
import torch
import torch.nn.functional as F
from transformers import (
CohereConfig,
Gemma2Config,
GraniteConfig,
MistralConfig,
)
try:
from transformers import FalconH1Config
except ImportError:
# Absent before transformers 4.53 (floor is 4.51.3), so not imported at module scope.
FalconH1Config = None
from transformers.modeling_outputs import BaseModelOutputWithPast
import pytest
import unsloth.models.llama as llama_module
import unsloth.models.mistral as mistral_module
from unsloth.models.llama import (
apply_logit_transforms,
CausalLM_fast_forward,
resolve_logit_transforms,
)
from unsloth.models.mistral import MistralForCausalLM_fast_forward
VOCAB_SIZE = 8
HIDDEN_SIZE = 4
BSZ = 2
Q_LEN = 4
LABELS = torch.tensor([[0, 1, 2, 3], [4, 5, 6, 7]])
def _hidden_states():
"""Deterministic without a seed, so the expected value survives a PRNG change."""
values = torch.arange(BSZ * Q_LEN * HIDDEN_SIZE, dtype = torch.float32)
return ((values % 7) - 3.0).reshape(BSZ, Q_LEN, HIDDEN_SIZE)
def _lm_head_weight():
values = torch.arange(VOCAB_SIZE * HIDDEN_SIZE, dtype = torch.float32)
return ((values % 5) - 2.0).reshape(VOCAB_SIZE, HIDDEN_SIZE)
class _FakeModel:
"""Stands in for the decoder stack: the branch under test only reads outputs[0]."""
def __init__(self, hidden_states):
self.hidden_states = hidden_states
def __call__(self, **kwargs):
return BaseModelOutputWithPast(
last_hidden_state = self.hidden_states,
past_key_values = None,
hidden_states = None,
attentions = None,
)
class _FakeCausalLM:
# mistral.py reads self.training before either loss branch, so pin it.
training = False
def __init__(self, config):
self.config = config
self.model = _FakeModel(_hidden_states())
self.lm_head = torch.nn.Linear(HIDDEN_SIZE, VOCAB_SIZE, bias = False)
with torch.no_grad():
self.lm_head.weight.copy_(_lm_head_weight())
def _fused_loss(
config,
monkeypatch,
forward = None,
):
monkeypatch.delenv("UNSLOTH_RETURN_LOGITS", raising = False)
monkeypatch.delenv("UNSLOTH_RETURN_HIDDEN_STATES", raising = False)
extra = {}
if forward is None:
forward = CausalLM_fast_forward(None)
else:
# mistral.py reads input_ids.shape before it reaches the decoder stack.
extra["input_ids"] = torch.zeros(BSZ, Q_LEN, dtype = torch.long)
output = forward(_FakeCausalLM(config), labels = LABELS.clone(), return_dict = True, **extra)
return output.loss.item()
def _reference_loss(scale, softcapping = 0.0):
"""Transformers' order: scale, soft cap, shift, cross entropy; computed in float64."""
logits = F.linear(_hidden_states().double(), _lm_head_weight().double())
logits = logits * scale
if softcapping:
logits = softcapping * torch.tanh(logits / softcapping)
shifted = torch.empty_like(LABELS)
shifted[..., :-1] = LABELS[..., 1:]
shifted[..., -1] = -100
loss = F.cross_entropy(logits.reshape(-1, VOCAB_SIZE), shifted.reshape(-1))
return loss.item()
def _config(cls, **kwargs):
"""Deferred: transformers validates strictly and per release, so eager construction
turns one unconstructable class into a collection error for the whole file."""
def build():
if cls is None:
pytest.skip("this transformers predates the model this case needs")
try:
return cls(
hidden_size = HIDDEN_SIZE,
intermediate_size = 2 * HIDDEN_SIZE,
num_hidden_layers = 1,
num_attention_heads = 2,
vocab_size = VOCAB_SIZE,
**kwargs,
)
except Exception as error:
pytest.skip(f"{cls.__name__} does not accept this shape here: {error}")
build.cls = cls
return build
@pytest.mark.parametrize(
"config,scale",
[
# Cohere / Command-R / Aya multiply, Granite 3 divides.
(_config(CohereConfig, logit_scale = 0.0625), 0.0625),
(_config(GraniteConfig, logits_scaling = 4.0), 1.0 / 4.0),
],
ids = ["cohere", "granite"],
)
def test_fused_ce_applies_the_configured_logit_scale(config, scale, monkeypatch):
config = config()
unscaled = _reference_loss(1.0) # 7.898634, what the fused branch returns unfixed
expected = _reference_loss(scale)
assert abs(expected - unscaled) > 1.0, "the fixture no longer separates the two losses"
assert _fused_loss(config, monkeypatch) == pytest.approx(expected, rel = 1e-6), (
f"fused CE dropped {config.model_type}'s logit scale: it returns the unscaled "
f"{unscaled:.6f} instead of {expected:.6f}"
)
def test_fused_ce_applies_falcon_h1_multiplier_once(monkeypatch):
"""The multiplier is folded into the hidden states, so it must not scale the logits too."""
config = _config(FalconH1Config, lm_head_multiplier = 3.0)()
assert _fused_loss(config, monkeypatch) == pytest.approx(_reference_loss(3.0), rel = 1e-6)
def test_mistral_fused_ce_reads_the_transforms_the_same_way(monkeypatch):
"""No shipped Mistral config carries a scale, so one is injected: the point is that
the call site forwards what the config holds, not that Mistral needs it."""
plain = _config(MistralConfig)()
assert _fused_loss(plain, monkeypatch, MistralForCausalLM_fast_forward) == pytest.approx(
_reference_loss(1.0),
rel = 1e-6,
), "plain Mistral carries no transform and must be unaffected"
scaled = _config(MistralConfig)()
scaled.logit_scale = 0.0625
assert _fused_loss(scaled, monkeypatch, MistralForCausalLM_fast_forward) == pytest.approx(
_reference_loss(0.0625),
rel = 1e-6,
), "mistral.py's fused branch dropped the configured logit scale"
@pytest.mark.parametrize(
"config,expected",
[
(_config(CohereConfig, logit_scale = 0.0625), (0, 0.0625, 0)),
(_config(GraniteConfig, logits_scaling = 4.0), (0, 0, 4.0)),
# The exact model_type test the fallback used to do missed the MoE spellings.
(_config(GraniteConfig, logits_scaling = 4.0, model_type = "granitemoe"), (0, 0, 4.0)),
(_config(FalconH1Config, lm_head_multiplier = 3.0), (0, 3.0, 0)),
],
ids = ["cohere", "granite", "granitemoe", "falcon_h1"],
)
def test_transforms_resolve_without_unsloth_zoo(config, expected, monkeypatch):
config = config()
"""The fallback arm runs whenever unsloth_zoo predates detect_logit_transforms."""
monkeypatch.setattr(llama_module, "detect_logit_transforms", None)
# Both arms agree on these configs, so prove the patch actually reached the code.
assert llama_module.detect_logit_transforms is None
assert resolve_logit_transforms(config) == pytest.approx(expected)
@pytest.mark.parametrize(
"model_type,fields",
[
("cohere", {"logit_scale": 0.0625}),
("granite", {"logits_scaling": 8.0}),
("granitemoehybrid", {"logits_scaling": 8.0}),
("falcon_h1", {"lm_head_multiplier": 3.0}),
# logits_scaling multiplies here (MuP), the opposite of Granite.
("hyperclovax", {"logits_scaling": 4.0}),
# logits_scaling scales the hidden states, so it is not a logit transform.
("minicpm3", {"logits_scaling": 4.0}),
("gemma2", {"final_logit_softcapping": 30.0}),
# The softcap spellings only the planner's table knows.
("recurrentgemma", {"logits_soft_cap": 30.0}),
("xlstm", {"output_logit_soft_cap": 30.0}),
("muse_glimmer", {"output_multiplier": 2.0, "final_logit_softcapping": 30.0}),
],
)
def test_both_resolver_arms_agree(model_type, fields, monkeypatch):
"""Old unsloth_zoo versions take the fallback resolver, which must match the detected transforms."""
def build():
config = MistralConfig(
hidden_size = HIDDEN_SIZE,
intermediate_size = 2 * HIDDEN_SIZE,
num_hidden_layers = 1,
num_attention_heads = 2,
vocab_size = VOCAB_SIZE,
)
config.model_type = model_type
for name, value in fields.items():
setattr(config, name, value)
return config
planner = llama_module.detect_logit_transforms
if planner is None:
pytest.skip("this unsloth_zoo predates detect_logit_transforms")
with_planner = resolve_logit_transforms(build())
monkeypatch.setattr(llama_module, "detect_logit_transforms", None)
without_planner = resolve_logit_transforms(build())
assert without_planner == pytest.approx(with_planner), (
f"the two arms disagree on {model_type}: the planner says {with_planner}, "
f"the fallback says {without_planner}"
)
@pytest.mark.parametrize(
"model_type,field",
[
("cohere", "logit_scale"),
("granite", "logits_scaling"),
("falcon_h1", "lm_head_multiplier"),
("gemma2", "final_logit_softcapping"),
],
)
def test_a_none_valued_field_resolves_to_zero(model_type, field, monkeypatch):
"""None must read as "off". A namespace, not a config: transformers 5 rejects None at
construction, but a checkpoint carrying one still loads into an older config."""
monkeypatch.setattr(llama_module, "detect_logit_transforms", None)
config = SimpleNamespace(model_type = model_type, **{field: None})
assert resolve_logit_transforms(config) == (0, 0, 0)
def test_a_none_model_type_does_not_raise(monkeypatch):
"""Remote-code configs do set model_type to None, and `in`-style reads must survive it."""
monkeypatch.setattr(llama_module, "detect_logit_transforms", None)
config = _config(CohereConfig, logit_scale = 0.0625)()
config.model_type = None
assert resolve_logit_transforms(config) == (0, 0.0625, 0)
def _inference_logits(model, forward):
extra = {"input_ids": torch.zeros(BSZ, Q_LEN, dtype = torch.long)} if forward else {}
forward = forward or CausalLM_fast_forward(None)
return forward(model, labels = None, return_dict = True, **extra).logits
_SCALE_CASES = [
(_config(CohereConfig, logit_scale = 0.0625), 0.0625, 0.0),
# Granite exercises the divisor fold, which the fused branch never takes.
(_config(GraniteConfig, logits_scaling = 4.0), 1.0 / 4.0, 0.0),
(_config(Gemma2Config, final_logit_softcapping = 4.0), 1.0, 4.0),
]
_SCALE_IDS = ["cohere", "granite", "gemma2_softcap"]
@pytest.mark.parametrize(
"forward", [None, MistralForCausalLM_fast_forward], ids = ["llama", "mistral"]
)
@pytest.mark.parametrize("config,scale,softcapping", _SCALE_CASES, ids = _SCALE_IDS)
def test_inference_logits_carry_the_same_transforms(config, scale, softcapping, forward):
config = config()
"""The labels-free branch returns the logits, so they must arrive already transformed."""
expected = F.linear(_hidden_states(), _lm_head_weight()) * scale
if softcapping:
expected = softcapping * torch.tanh(expected / softcapping)
model = _FakeCausalLM(config)
assert torch.allclose(_inference_logits(model, forward), expected, atol = 1e-5)
# Applied in place, so a second call on the same model must not compound them.
assert torch.allclose(_inference_logits(model, forward), expected, atol = 1e-5)
@pytest.mark.parametrize(
"module,forward",
[(llama_module, None), (mistral_module, MistralForCausalLM_fast_forward)],
ids = ["llama", "mistral"],
)
@pytest.mark.parametrize("config,scale,softcapping", _SCALE_CASES, ids = _SCALE_IDS)
def test_materialized_branch_is_handed_the_same_transforms(
config, scale, softcapping, module, forward, monkeypatch
):
"""The eager branch needs Triton, so assert on what it hands the kernel. Each module
has its own binding via `from .llama import *`, so the spy goes on the call site's."""
config = config()
calls = []
def spy(**kwargs):
calls.append(kwargs)
return torch.zeros((), requires_grad = True)
monkeypatch.setattr(module, "fast_cross_entropy_loss", spy)
monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")
extra = {"input_ids": torch.zeros(BSZ, Q_LEN, dtype = torch.long)} if forward else {}
(forward or CausalLM_fast_forward(None))(
_FakeCausalLM(config),
labels = LABELS.clone(),
return_dict = True,
**extra,
)
assert len(calls) == 1, "the eager branch did not run"
assert calls[0]["logit_scaling"] == pytest.approx(scale if scale != 1.0 else 0)
assert calls[0]["logit_softcapping"] == pytest.approx(softcapping)
@pytest.mark.parametrize("grad", [True, False], ids = ["grad", "no_grad"])
@pytest.mark.parametrize(
"module,forward",
[(llama_module, None), (mistral_module, MistralForCausalLM_fast_forward)],
ids = ["llama", "mistral"],
)
@pytest.mark.parametrize("config,scale,softcapping", _SCALE_CASES, ids = _SCALE_IDS)
def test_labeled_call_returns_the_transformed_logits(
config, scale, softcapping, module, forward, grad, monkeypatch
):
"""The kernel transforms the raw logits itself, so the returned ones are transformed
separately, and only after the kernel has read them: without grad that is in place."""
raw = F.linear(_hidden_states(), _lm_head_weight())
expected = raw * scale
if softcapping:
expected = softcapping * torch.tanh(expected / softcapping)
seen = []
def spy(**kwargs):
seen.append(kwargs["logits"].detach().clone())
return torch.zeros((), requires_grad = True)
monkeypatch.setattr(module, "fast_cross_entropy_loss", spy)
monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")
extra = {"input_ids": torch.zeros(BSZ, Q_LEN, dtype = torch.long)} if forward else {}
with torch.set_grad_enabled(grad):
output = (forward or CausalLM_fast_forward(None))(
_FakeCausalLM(config()),
labels = LABELS.clone(),
return_dict = True,
**extra,
)
assert torch.allclose(seen[0], raw, atol = 1e-5), "the kernel would transform these twice"
assert torch.allclose(output.logits, expected, atol = 1e-5)
assert output.logits.requires_grad == grad
def test_transforms_are_applied_scale_first_then_soft_cap():
"""tanh does not commute with scaling, and no shipped family does both, so nothing
above separates the two orders."""
logits = F.linear(_hidden_states(), _lm_head_weight())
scale, softcapping = 0.0625, 4.0
scale_first = softcapping * torch.tanh((logits * scale) / softcapping)
cap_first = softcapping * torch.tanh(logits / softcapping) * scale
assert not torch.allclose(scale_first, cap_first), "the fixture no longer separates the orders"
assert torch.allclose(
apply_logit_transforms(logits.clone(), softcapping, scale),
scale_first,
atol = 1e-6,
)