Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix DPO on models split across GPUs with TRL 1.15's fused LM head (#13641)
Daniel Han3370575b80cc
diff --git a/tests/test_trl_fused_lm_head_devices.py b/tests/test_trl_fused_lm_head_devices.pynew file mode 100644index 00000000000..1193a1f7e95--- /dev/null+++ b/tests/test_trl_fused_lm_head_devices.py@@ -0,0 +1,229 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""CPU checks for patch_trl_fused_lm_head_devices (loaded from rl.py source, TRL's utils replaced by a fake)."""++from __future__ import annotations++import ast+import functools+import os+import sys+import types+from collections import OrderedDict++import pytest+import torch+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py++REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))+RL_SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl.py")+NAMES = (+ "_spans_several_devices",+ "_first_tensor_device",+ "_ApplyOnInputDevice",+ "patch_trl_fused_lm_head_devices",+)+++class _Outputs(OrderedDict):+ """ModelOutput stand-in: a dict whose items TRL reads by attribute."""++ __getattr__ = OrderedDict.__getitem__+++def _load():+ src = open(RL_SOURCE_PATH, encoding = "utf-8").read()+ ns = {"torch": torch, "functools": functools, "sys": sys}+ for node in ast.parse(src).body:+ if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name in NAMES:+ exec(compile(ast.Module([node], []), RL_SOURCE_PATH, "exec"), ns)+ assert all(n in ns for n in NAMES)+ return ns+++class _Model(torch.nn.Module):+ def __init__(self, device_map):+ super().__init__()+ if device_map is not None:+ self.hf_device_map = device_map++ def forward(+ self,+ input_ids = None,+ **kwargs,+ ):+ return "original"+++def _fake_add_fused_lm_head(model, *args, **kwargs):+ # Like TRL: the head output lives on the head's device (CPU here), whatever device the inputs came on.+ def _fused_forward(+ self,+ *f_args,+ fused_lm_head = False,+ labels = None,+ **f_kwargs,+ ):+ if not fused_lm_head:+ return "unfused"+ rows, cols = labels.shape+ return _Outputs(+ loss = torch.zeros(()),+ log_probs = torch.zeros(rows, cols),+ entropy = None,+ label_mask = torch.ones(rows, cols, dtype = torch.bool),+ )++ model.forward = types.MethodType(_fused_forward, model)+++class _Kernel:+ calls = []++ @staticmethod+ def apply(hidden, *args):+ _Kernel.calls.append(+ (hidden.device, torch.cuda.current_device() if hidden.device.type == "cuda" else None)+ )+ return hidden+++def fake_trl(monkeypatch):+ utils = types.ModuleType("trl.trainer.utils")+ utils.add_fused_lm_head = _fake_add_fused_lm_head+ utils._ChunkedLogProbFunction = _Kernel+ dpo = types.ModuleType("trl.trainer.dpo_trainer")+ dpo.add_fused_lm_head = _fake_add_fused_lm_head+ unrelated = types.ModuleType("trl.trainer.other")+ for name, module in (+ ("trl", types.ModuleType("trl")),+ ("trl.trainer", types.ModuleType("trl.trainer")),+ ("trl.trainer.utils", utils),+ ("trl.trainer.dpo_trainer", dpo),+ ("trl.trainer.other", unrelated),+ ):+ monkeypatch.setitem(sys.modules, name, module)+ sys.modules["trl"].trainer = sys.modules["trl.trainer"]+ sys.modules["trl.trainer"].utils = utils+ return utils, dpo, unrelated+++def test_rebinds_every_trl_module_and_is_idempotent(fake_trl):+ utils, dpo, unrelated = fake_trl+ ns = _load()+ ns["patch_trl_fused_lm_head_devices"]()+ patched = utils.add_fused_lm_head+ assert patched is not _fake_add_fused_lm_head+ assert patched._unsloth_device_patched+ assert dpo.add_fused_lm_head is patched+ assert not hasattr(unrelated, "add_fused_lm_head")+ ns["patch_trl_fused_lm_head_devices"]()+ assert utils.add_fused_lm_head is patched++[email protected]("device_map", [None, {"": 0}, {"model": 0, "lm_head": 0}])+def test_single_device_model_keeps_trl_forward(fake_trl, device_map):+ utils, _, _ = fake_trl+ _load()["patch_trl_fused_lm_head_devices"]()+ model = _Model(device_map)+ utils.add_fused_lm_head(model)+ assert isinstance(model.forward, types.MethodType)+ assert model.forward.__func__.__name__ == "_fused_forward"+ assert utils._ChunkedLogProbFunction is _Kernel+++def test_split_model_returns_outputs_on_input_device(fake_trl):+ utils, _, _ = fake_trl+ _load()["patch_trl_fused_lm_head_devices"]()+ model = _Model({"model.embed_tokens": 0, "model.layers.0": 1, "lm_head": 1})+ utils.add_fused_lm_head(model)+ # meta stands in for the input card: the fake head answers on the CPU, like a head on another card.+ input_ids = torch.empty(2, 5, dtype = torch.long, device = "meta")+ labels = torch.zeros(2, 5, dtype = torch.long)+ out = model(input_ids = input_ids, labels = labels, fused_lm_head = True)+ assert out.log_probs.device.type == "meta"+ assert out.label_mask.device.type == "meta"+ assert out.loss.device.type == "meta"+ assert out.entropy is None+ assert model(input_ids = input_ids) == "unfused"+ assert utils._ChunkedLogProbFunction.function is _Kernel+ shim = utils._ChunkedLogProbFunction+ utils.add_fused_lm_head(_Model({"model": 0, "lm_head": 1}))+ assert utils._ChunkedLogProbFunction is shim+++def test_kernel_shim_passes_cpu_tensors_through():+ ns = _load()+ _Kernel.calls.clear()+ hidden = torch.ones(2, 3)+ assert ns["_ApplyOnInputDevice"](_Kernel).apply(hidden, 1, 2) is hidden+ assert _Kernel.calls[-1][0].type == "cpu"++[email protected]("device_type", ["cuda", "xpu"])+def test_kernel_shim_switches_to_the_hidden_device(monkeypatch, device_type):+ ns = _load()+ backend = getattr(torch, device_type)+ entered = []++ class _Device:+ def __init__(self, device):+ self.device = device++ def __enter__(self):+ entered.append(self.device)++ def __exit__(self, *exc):+ entered.append(None)++ monkeypatch.setattr(backend, "current_device", lambda: 0)+ monkeypatch.setattr(backend, "device", _Device)+ hidden = types.SimpleNamespace(device = torch.device(device_type, 1))+ kernel = types.SimpleNamespace(apply = lambda h, *a: (entered[-1] if entered else "none", a))+ assert ns["_ApplyOnInputDevice"](kernel).apply(hidden, 7) == (+ torch.device(device_type, 1),+ (7,),+ )+ assert entered == [torch.device(device_type, 1), None]+ entered.clear()+ hidden.device = torch.device(device_type, 0)+ assert ns["_ApplyOnInputDevice"](kernel).apply(hidden) == ("none", ())+ assert entered == []+++ not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs two CUDA devices"+)+def test_kernel_shim_runs_on_the_hidden_device():+ ns = _load()+ _Kernel.calls.clear()+ with torch.cuda.device(0):+ ns["_ApplyOnInputDevice"](_Kernel).apply(torch.ones(2, 3, device = "cuda:1"))+ assert torch.cuda.current_device() == 0+ assert _Kernel.calls[-1] == (torch.device("cuda", 1), 1)+++def test_missing_trl_helper_is_a_no_op(monkeypatch):+ utils = types.ModuleType("trl.trainer.utils")+ monkeypatch.setitem(sys.modules, "trl", types.ModuleType("trl"))+ monkeypatch.setitem(sys.modules, "trl.trainer", types.ModuleType("trl.trainer"))+ monkeypatch.setitem(sys.modules, "trl.trainer.utils", utils)+ sys.modules["trl"].trainer = sys.modules["trl.trainer"]+ sys.modules["trl.trainer"].utils = utils+ _load()["patch_trl_fused_lm_head_devices"]()+ assert not hasattr(utils, "add_fused_lm_head")diff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 02ece8e4dc5..34cb590bf67 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -4256,6 +4256,83 @@ def patch_trl_disable_gradient_checkpointing():return+def _spans_several_devices(model):+ device_map = getattr(model, "hf_device_map", None)+ return isinstance(device_map, dict) and len({str(d) for d in device_map.values()}) > 1+++def _first_tensor_device(args, kwargs):+ for key in ("input_ids", "inputs_embeds", "attention_mask", "labels"):+ value = kwargs.get(key)+ if isinstance(value, torch.Tensor):+ return value.device+ for value in list(args) + list(kwargs.values()):+ if isinstance(value, torch.Tensor):+ return value.device+ return None+++class _ApplyOnInputDevice:+ # Triton launches on the current device, not the tensors': off it, TRL's chunked log-prob kernel returns inf.+ _unsloth_device_patched = True++ def __init__(self, function):+ self.function = function++ def apply(self, hidden, *args):+ device = hidden.device+ backend = getattr(torch, device.type, None) if device.type in ("cuda", "xpu") else None+ if backend is None or device.index is None or device.index == backend.current_device():+ return self.function.apply(hidden, *args)+ with backend.device(device):+ return self.function.apply(hidden, *args)+++def patch_trl_fused_lm_head_devices():+ # TRL >= 1.15's fused head drops accelerate's io_same_device hook: on a split model log_probs stay on lm_head's card (#4030).+ try:+ import trl.trainer.utils as _tu+ except ImportError:+ return+ original = getattr(_tu, "add_fused_lm_head", None)+ if original is None or getattr(original, "_unsloth_device_patched", False):+ return++ @functools.wraps(original)+ def add_fused_lm_head(model, *args, **kwargs):+ result = original(model, *args, **kwargs)+ if not _spans_several_devices(model):+ return result+ kernel = getattr(_tu, "_ChunkedLogProbFunction", None)+ if kernel is not None and not getattr(kernel, "_unsloth_device_patched", False):+ _tu._ChunkedLogProbFunction = _ApplyOnInputDevice(kernel)+ fused_forward = model.forward++ @functools.wraps(fused_forward)+ def forward(*f_args, **f_kwargs):+ outputs = fused_forward(*f_args, **f_kwargs)+ if not f_kwargs.get("fused_lm_head", False):+ return outputs+ device = _first_tensor_device(f_args, f_kwargs)+ if device is None:+ return outputs+ for key, value in outputs.items():+ if isinstance(value, torch.Tensor) and value.device != device:+ outputs[key] = value.to(device)+ return outputs++ model.forward = forward+ return result++ add_fused_lm_head._unsloth_device_patched = True+ _tu.add_fused_lm_head = add_fused_lm_head+ for _mod_name, _mod in list(sys.modules.items()):+ if _mod is None or not _mod_name.startswith("trl."):+ continue+ if getattr(_mod, "add_fused_lm_head", None) is original:+ setattr(_mod, "add_fused_lm_head", add_fused_lm_head)++def patch_trl_openenv():for function in RL_ADDITIONAL_FUNCTIONS["openenv"]:logger.info(f"Unsloth: Patching trl openenv with function: {function.__name__}")@@ -4769,6 +4846,10 @@ def PatchFastRL(algorithm = None, FastLanguageModel = None):return# Install the disable_gradient_checkpointing noop BEFORE patch_trl_rl_trainers, which imports more trl.* submodules: anything imported after the sys.modules walk keeps the old binding.patch_trl_disable_gradient_checkpointing()+ try:+ patch_trl_fused_lm_head_devices()+ except Exception as e:+ logger.info(f"Unsloth: Could not patch trl add_fused_lm_head for split models: {e}")# Also before patch_trl_rl_trainers, so the generated trainers import the wrapped helpers.try:from ._trl_tool_templates import patch_trl_tool_chat_templates