Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
"""Full fine-tuning with beta > 0 scores a separate ref_model: its log-probs need its own head, not the policy's."""
import ast
import textwrap
from pathlib import Path
from types import SimpleNamespace
import pytest
torch = pytest.importorskip("torch")
_SOURCE_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"
def _lm_head_assignment():
source = _SOURCE_PATH.read_text(encoding = "utf-8")
(function,) = [
node
for node in ast.walk(ast.parse(source))
if isinstance(node, ast.FunctionDef) and node.name == "_get_per_token_logps_and_entropies"
]
segments = {}
for node in ast.walk(function):
if isinstance(node, ast.Assign):
for target in node.targets:
if isinstance(target, ast.Name) and target.id in (
"output_embeddings",
"lm_head",
"lm_head_bias",
):
segments.setdefault(
(node.lineno, target.id), ast.get_source_segment(source, node)
)
# First binding of each name, in source order: the head lookup, before any fallback rebinding.
first = {}
for (_, name), segment in sorted(segments.items()):
first.setdefault(name, segment)
code = "\n".join(
textwrap.dedent(first[name])
for name in ("output_embeddings", "lm_head", "lm_head_bias")
if name in first
)
return compile(code, "<lm_head>", "exec")
def _causal_lm(seed, bias = False):
torch.manual_seed(seed)
head = torch.nn.Linear(8, 16, bias = bias)
return SimpleNamespace(get_output_embeddings = lambda: head)
def test_reference_model_scored_with_its_own_head():
policy, ref = _causal_lm(0), _causal_lm(1)
namespace = {"self": SimpleNamespace(model = policy), "unwrapped_model": ref}
exec(_lm_head_assignment(), namespace)
assert namespace["lm_head"] is ref.get_output_embeddings().weight
def test_reference_model_scored_with_its_own_head_bias():
policy, ref = _causal_lm(0, bias = True), _causal_lm(1, bias = True)
namespace = {"self": SimpleNamespace(model = policy), "unwrapped_model": ref}
exec(_lm_head_assignment(), namespace)
assert namespace["lm_head"] is ref.get_output_embeddings().weight
assert namespace["lm_head_bias"] is ref.get_output_embeddings().bias