Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team.
"""ORPO / CPO hidden-state scoring traces as one graph (no data-dependent branch, no boolean slice assignment) and
matches TRL's math in fp64, eager and compiled. CPU only: the helper is lifted from source via ast and the zoo
kernels are replaced by dense references, so no package import, model or GPU is needed."""
from __future__ import annotations
import ast
import os
import types
from pathlib import Path
import pytest
torch = pytest.importorskip("torch")
F = torch.nn.functional
SOURCE = Path(__file__).resolve().parents[2] / "unsloth" / "models" / "rl_replacements.py"
VOCAB, HIDDEN, SEQ, PAIRS = 23, 8, 7, 2
def _transform(logits, multiply, divide, softcap):
if multiply:
logits = logits * multiply
if divide:
logits = logits / divide
if softcap:
logits = softcap * torch.tanh(logits / softcap)
return logits
def _logits(
hidden,
weight,
bias,
multiply = 0.0,
divide = 0.0,
softcap = 0.0,
):
return _transform(F.linear(hidden, weight, bias), multiply, divide, softcap)
def _chunked(
hidden,
weight,
index,
return_mean_logits = False,
lm_head_bias = None,
logit_scale_multiply = 0.0,
logit_scale_divide = 0.0,
logit_softcapping = 0.0,
**_,
):
logits = _logits(
hidden, weight, lm_head_bias, logit_scale_multiply, logit_scale_divide, logit_softcapping
)
logps = logits.log_softmax(-1).gather(-1, index.unsqueeze(-1)).squeeze(-1)
return (logps, logits.mean(-1)) if return_mean_logits else logps
def _masked(
hidden,
weight,
labels,
mask = None,
**kwargs,
):
return torch.where(mask, _chunked(hidden, weight, labels, **kwargs), 0.0)
def _mean_logits(hidden, weight, bias, multiply, divide, softcap):
return _logits(hidden, weight, bias, multiply, divide, softcap).mean(-1)
TRANSFORMS = (0.7, 0.0, 3.0)
def _load(masked_kernel):
tree = ast.parse(SOURCE.read_text(encoding = "utf-8"))
node = next(
n
for n in tree.body
if isinstance(n, ast.FunctionDef) and n.name == "_unsloth_pref_chunked_forward"
)
ns = {
"torch": torch,
"os": os,
"unsloth_chunked_logprobs": _chunked,
"_unsloth_pref_masked_logprobs": _masked if masked_kernel else None,
"_unsloth_linear_mean_logits": _mean_logits if masked_kernel else None,
"_unsloth_pref_logit_transforms": lambda model: TRANSFORMS,
"_unsloth_pref_returns_hidden_states": lambda model, tensor, weight: True,
}
exec(compile(ast.Module(body = [node], type_ignores = []), str(SOURCE), "exec"), ns)
return ns["_unsloth_pref_chunked_forward"]
class _Trainer:
def __init__(
self,
cpo_alpha = 1.0,
loss_type = "sigmoid",
):
self.cpo_alpha = cpo_alpha
self.loss_type = loss_type
self.accelerator = types.SimpleNamespace(
unwrap_model = lambda m: m, device = torch.device("cpu")
)
def _batch(split):
g = torch.Generator().manual_seed(0)
rows = 2 * PAIRS
input_ids = torch.randint(1, VOCAB, (rows, SEQ), generator = g)
attention_mask = torch.ones(rows, SEQ, dtype = torch.long)
attention_mask[0, -2:] = 0 # right padding on a chosen row
attention_mask[PAIRS + 1, -1:] = 0 # and on a rejected row
labels = torch.where(attention_mask == 1, input_ids, -100)
labels[:, :3] = -100 # prompt
if split:
# A completion token scored against a label other than its input id: ORPO needs a second NLL pass.
labels[0, 4] = (input_ids[0, 4] + 1) % VOCAB
states = torch.randn(rows, SEQ, HIDDEN, generator = g, dtype = torch.float64, requires_grad = True)
head = torch.nn.Linear(HIDDEN, VOCAB, bias = True).double()
with torch.no_grad():
head.weight.copy_(torch.randn(VOCAB, HIDDEN, generator = g, dtype = torch.float64))
head.bias.copy_(torch.randn(VOCAB, generator = g, dtype = torch.float64))
batch = {
"concatenated_input_ids": input_ids,
"concatenated_labels": labels,
"concatenated_attention_mask": attention_mask,
}
return head, batch, states
def _oracle(kind, trainer, head, batch, states):
"""TRL's ORPO / CPO concatenated_forward math on dense logits."""
logits = _logits(states, head.weight, head.bias, *TRANSFORMS)
labels = batch["concatenated_labels"]
shifted, scored = labels[:, 1:], logits[:, :-1]
mask = shifted != -100
per_token = scored.log_softmax(-1).gather(-1, shifted.clamp(min = 0).unsqueeze(-1)).squeeze(-1)
logps = (per_token * mask).sum(-1)
if kind == "orpo" or trainer.loss_type in ("ipo", "simpo"):
logps = logps / mask.sum(-1)
if kind == "orpo":
nll_labels = torch.where(
batch["concatenated_attention_mask"] == 1, batch["concatenated_input_ids"], -100
)
nll = F.cross_entropy(scored[:PAIRS].flatten(0, 1), nll_labels[:PAIRS, 1:].flatten())
means = scored.mean(-1)
else:
nll = F.cross_entropy(scored[:PAIRS].flatten(0, 1), shifted[:PAIRS].flatten())
if trainer.cpo_alpha == 0:
nll = torch.tensor(0.0) # TRL: torch.tensor(0.0).to(self.accelerator.device)
means = logits.mean(-1)
return logps[:PAIRS], logps[PAIRS:], means[:PAIRS], means[PAIRS:], nll
def _scalar(outputs):
weights = (1.0, -0.5, 0.25, 0.125, 2.0)
return sum(w * o.sum() for w, o in zip(weights, outputs))
def _grads(fn, head, states):
head.zero_grad(set_to_none = True)
states.grad = None
outputs = fn()
_scalar(outputs).backward()
return [o.detach() for o in outputs], [
states.grad.clone(),
head.weight.grad.clone(),
head.bias.grad.clone(),
]
CASES = [
("orpo", {}, False),
("orpo", {}, True),
("cpo", {"cpo_alpha": 1.0, "loss_type": "sigmoid"}, False),
("cpo", {"cpo_alpha": 1.0, "loss_type": "simpo"}, True),
("cpo", {"cpo_alpha": 0.0, "loss_type": "sigmoid"}, False),
]
@pytest.mark.parametrize("masked_kernel", [True, False], ids = ["masked", "dense"])
@pytest.mark.parametrize("kind,kwargs,split", CASES)
def test_traces_without_graph_breaks_and_matches_trl(kind, kwargs, split, masked_kernel):
forward = _load(masked_kernel)
trainer = _Trainer(**kwargs)
head, batch, states = _batch(split)
def body(states, input_ids, labels, attention_mask):
concatenated = {
"concatenated_input_ids": input_ids,
"concatenated_labels": labels,
"concatenated_attention_mask": attention_mask,
}
outputs, _ = forward(trainer, head, head, concatenated, PAIRS, states, kind)
return outputs
args = (
states,
batch["concatenated_input_ids"],
batch["concatenated_labels"],
batch["concatenated_attention_mask"],
)
torch._dynamo.reset()
explained = torch._dynamo.explain(body)(*args)
assert explained.graph_break_count == 0, explained.break_reasons
assert explained.graph_count == 1
want, want_grads = _grads(lambda: _oracle(kind, trainer, head, batch, states), head, states)
got, got_grads = _grads(lambda: body(*args), head, states)
for a, b in zip(got + got_grads, want + want_grads):
torch.testing.assert_close(a, b, rtol = 1e-10, atol = 1e-12)
for dynamic in (False, True):
torch._dynamo.reset()
compiled = torch.compile(body, fullgraph = True, backend = "aot_eager", dynamic = dynamic)
out, grads = _grads(lambda: compiled(*args), head, states)
for a, b in zip(out + grads, got + got_grads):
torch.testing.assert_close(a, b, rtol = 1e-10, atol = 1e-12)
torch._dynamo.reset()
@pytest.mark.parametrize("dynamic", [False, True, None], ids = ["static", "dynamic", "auto"])
def test_engagement_counter_compiles_once(dynamic):
from torch._dynamo.testing import CompileCounterWithBackend
forward = _load(True)
trainer = _Trainer()
head, batch, states = _batch(False)
def body(states):
outputs, _ = forward(trainer, head, head, batch, PAIRS, states, "cpo")
return _scalar(outputs)
body(states).backward()
assert trainer._unsloth_pref_chunked_calls == 1
torch._dynamo.reset()
counter = CompileCounterWithBackend("aot_eager")
compiled = torch.compile(body, fullgraph = True, backend = counter, dynamic = dynamic)
compiled(states).backward()
first = counter.frame_count
for _ in range(9):
compiled(states).backward()
assert counter.frame_count == first
body(states).backward()
assert trainer._unsloth_pref_chunked_calls == 2
torch._dynamo.reset()
def test_every_engagement_counter_skips_tracing():
import re
root = SOURCE.parent
sites = 0
for path in (SOURCE, root / "_fused_rl_logprobs.py"):
tree = ast.parse(path.read_text(encoding = "utf-8"))
guarded = set()
for node in ast.walk(tree):
if (
isinstance(node, ast.If)
and ast.unparse(node.test) == "not torch.compiler.is_compiling()"
):
guarded.update(id(n) for n in ast.walk(node))
for node in ast.walk(tree):
if isinstance(node, ast.Assign) and re.search(
r"_unsloth_\w+_calls", ast.unparse(node.targets[0])
):
sites += 1
assert id(node) in guarded, f"{path.name}:{node.lineno} counts inside traced code"
assert sites >= 10