Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
[bugfix] Snapshot in-memory DPO policies for the default reference (#15537)
cherry77-cloud24e57363a441
diff --git a/tests/test_dpo_reference_snapshot.py b/tests/test_dpo_reference_snapshot.pynew file mode 100644index 00000000000..1870f4543f6--- /dev/null+++ b/tests/test_dpo_reference_snapshot.py@@ -0,0 +1,225 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""DPO default reference: an in-memory policy is snapshotted, not reloaded from its checkpoint path."""++import ast+import pathlib+import re+import textwrap+import types++import pytest+import torch+++def _hooks():+ path = pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ wanted = {+ "_insert_after_docstring",+ "_unsloth_dpo_reference_loadable",+ "_unsloth_dpo_copy_policy_into_reference",+ "dpo_trainer_reference_snapshot",+ }+ nodes = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in wanted]+ namespace = {"re": re, "textwrap": textwrap, "torch": torch}+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(path), "exec"), namespace)+ return namespace+++_NS = _hooks()+hook = _NS["dpo_trainer_reference_snapshot"]++# Shape of TRL >= 0.29 DPOTrainer.__init__ around the default reference.+TRL_INIT = '''+ def __init__(self, model, ref_model = None, args = None):+ """Docstring."""+ if isinstance(model, str):+ model = create_model_from_path(model)+ self.model = model+ if ref_model is None:+ if is_peft_model(self.model):+ self.ref_model = None+ else:+ ref_model_init_kwargs = dict(args or {}) # copy+ ref_model_init_kwargs.setdefault("trust_remote_code", False)+ ref_model_path = get_config_model_id(self.model.config)+ self.ref_model = create_model_from_path(ref_model_path, **ref_model_init_kwargs)+ else:+ self.ref_model = ref_model+ warnings.warn("later TRL warning") # e.g. use_liger_kernel, via TRL's module-level import+'''+++class _Policy(torch.nn.Module):+ config = "ckpt"++ def __init__(+ self,+ out = 2,+ tied = False,+ offload = False,+ ):+ super().__init__()+ torch.manual_seed(0) # the checkpoint's weights+ self.lin = torch.nn.Linear(2, 2)+ self.head = torch.nn.Linear(2, out)+ if tied:+ self.head.weight = self.lin.weight+ if offload == "meta": # one meta tensor: two would also differ in tying (both data_ptr 0)+ self.head.weight = torch.nn.Parameter(self.head.weight.to("meta"))+ elif offload == "quantized": # e.g. TorchAO Int8Tensor from model_init_kwargs+ self.head.weight = torch.nn.Parameter(self.head.weight.detach().as_subclass(_Shard))+++class _Shard(torch.Tensor): # stands in for DTensor (FSDP2 / tensor parallel)+ pass+++class _ShardedPolicy(_Policy):+ def __init__(self):+ super().__init__()+ self.lin.weight = torch.nn.Parameter(self.lin.weight.detach().as_subclass(_Shard))+++def _run(+ source,+ model,+ monkeypatch,+ zero3 = False,+ out = 2,+ ref_model = None,+ tied = False,+ offload = False,+ dtype = torch.float32,+):+ import transformers.integrations.deepspeed as ds++ monkeypatch.setattr(ds, "is_deepspeed_zero3_enabled", lambda: zero3)+ namespace = {+ "_unsloth_dpo_reference_loadable": _NS["_unsloth_dpo_reference_loadable"],+ "_unsloth_dpo_copy_policy_into_reference": _NS["_unsloth_dpo_copy_policy_into_reference"],+ "create_model_from_path": lambda path, **kwargs: _Policy(out, tied, offload).to(dtype),+ "is_peft_model": lambda m: False,+ "get_config_model_id": lambda config: config,+ "torch": torch,+ "warnings": __import__("warnings"),+ }+ exec(textwrap.dedent(source), namespace)+ trainer = types.SimpleNamespace()+ with pytest.warns(UserWarning, match = "later TRL warning"):+ namespace["__init__"](trainer, model, ref_model)+ return trainer.ref_model+++def _trained(cls = _Policy):+ policy = cls()+ with torch.no_grad():+ policy.lin.weight.add_(1.0)+ return policy+++def test_instance_reference_is_in_memory_policy(monkeypatch):+ policy = _trained()+ # Unsloth patches instance forwards with closures over the policy's modules (block-compiled experts).+ policy.lin.forward = torch.compiler.disable(policy.lin.forward)+ ref = _run(hook("__init__", TRL_INIT), policy, monkeypatch)+ assert ref is not policy and torch.equal(ref.lin.weight, policy.lin.weight)+ x = torch.ones(1, 2)+ before = ref.lin(x)+ with torch.no_grad():+ policy.lin.weight.add_(1.0)+ assert torch.equal(ref.lin(x), before)+ ref = _run(TRL_INIT, _trained(), monkeypatch)+ assert torch.equal(ref.lin.weight, _Policy().lin.weight)++[email protected]("model, zero3", [("hub/id", False), (None, True)])+def test_path_and_zero3_keep_checkpoint(monkeypatch, model, zero3):+ model = _trained() if model is None else model+ ref = _run(hook("__init__", TRL_INIT), model, monkeypatch, zero3 = zero3)+ assert torch.equal(ref.lin.weight, _Policy().lin.weight)+++def test_reference_keeps_policy_dtypes(monkeypatch):+ # BF16 full finetune keeps FP32 norms; the reference loads in the policy's (BF16) dtype.+ policy = _trained().to(torch.bfloat16)+ policy.lin.float()+ with torch.no_grad():+ policy.lin.weight.add_(1e-3) # below BF16 resolution at ~1+ ref = _run(hook("__init__", TRL_INIT), policy, monkeypatch, dtype = torch.bfloat16)+ for name, value in policy.state_dict().items():+ assert ref.state_dict()[name].dtype == value.dtype and torch.equal(+ ref.state_dict()[name], value+ ), name+++def test_explicit_reference_kept(monkeypatch):+ explicit = _Policy()+ assert _run(hook("__init__", TRL_INIT), _trained(), monkeypatch, ref_model = explicit) is explicit+++ "policy, out, tied, offload",+ [+ (_Policy, 3, False, False),+ (_Policy, 2, True, False),+ (_ShardedPolicy, 2, False, False),+ (_Policy, 2, False, "meta"),+ (_Policy, 2, False, "quantized"),+ ],+ ids = [+ "resized",+ "untied_policy_tied_checkpoint",+ "sharded",+ "offloaded_reference",+ "quantized_reference",+ ],+)+def test_mismatched_reference_keeps_whole_checkpoint(monkeypatch, policy, out, tied, offload):+ with pytest.warns(UserWarning, match = "do not match its checkpoint"):+ ref = _run(+ hook("__init__", TRL_INIT),+ _trained(policy),+ monkeypatch,+ out = out,+ tied = tied,+ offload = offload,+ )+ checkpoint = _Policy(out, tied, offload)+ for name, value in checkpoint.state_dict().items():+ if not value.is_meta:+ assert torch.equal(ref.state_dict()[name], value), name+++def test_rewrite_scope():+ patched = hook("__init__", TRL_INIT)+ assert patched != TRL_INIT and hook("__init__", patched) == patched+ assert ".state_dict()" not in patched+ assert hook("compute_loss", TRL_INIT) == TRL_INIT+ legacy = TRL_INIT.replace(+ "create_model_from_path(ref_model_path, **ref_model_init_kwargs)",+ "create_reference_model(model)",+ )+ assert hook("__init__", legacy) == legacy+++def test_installed_trl_rewrites_and_compiles():+ trl = pytest.importorskip("trl")+ import trl.trainer.dpo_trainer as installed++ # From the file: conftest imports unsloth, which swaps in the compiled UnslothDPOTrainer.+ text = pathlib.Path(installed.__file__).read_text(encoding = "utf-8")+ cls = next(+ n for n in ast.parse(text).body if isinstance(n, ast.ClassDef) and n.name == "DPOTrainer"+ )+ init = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "__init__")+ source = ast.get_source_segment(text, init, padded = True)+ if "create_model_from_path(ref_model_path" not in source:+ pytest.skip(f"TRL {trl.__version__} already copies the policy")+ patched = hook("__init__", source)+ assert "_unsloth_dpo_copy_policy_into_reference(self.ref_model" in patched+ # rl.py's _strip_vllm_weight_loads deletes `.state_dict()` from every generated trainer.+ assert ".state_dict()" not in patched+ compile(textwrap.dedent(patched), "<patched>", "exec")diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 3b0ef4ee7f4..95ac73a8020 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -738,6 +738,81 @@ def dpo_trainer_prepare_dataset(function_name, function):return function+def _unsloth_dpo_reference_loadable(policy_sd, ref_sd):+ # Plain materialised tensors only (not DTensor / quantized / meta), equal shapes, equal tying:+ # an untied policy loaded into a tied reference writes twice into one tensor.+ if policy_sd.keys() != ref_sd.keys():+ return False+ policy_first, ref_first = {}, {}+ for key, value in policy_sd.items():+ ref = ref_sd[key]+ if (+ type(value) is not torch.Tensor+ or type(ref) is not torch.Tensor+ or value.shape != ref.shape+ or value.is_meta+ or ref.is_meta+ ):+ return False+ if policy_first.setdefault(value.data_ptr(), key) != ref_first.setdefault(+ ref.data_ptr(), key+ ):+ return False+ return True+++def _unsloth_dpo_copy_policy_into_reference(ref_model, policy_sd):+ # Recast first: load_state_dict would round FP32 norm updates into a BF16 reference; .data keeps ties.+ tensors = dict(ref_model.named_parameters(remove_duplicate = False))+ tensors.update(ref_model.named_buffers(remove_duplicate = False))+ for key, value in policy_sd.items():+ tensor = tensors.get(key)+ if tensor is not None and tensor.dtype != value.dtype:+ tensor.data = tensor.data.to(value.dtype)+ ref_model.load_state_dict(policy_sd)+++def dpo_trainer_reference_snapshot(function_name, function):+ if function_name != "__init__" or "_unsloth_dpo_model_is_instance" in function:+ return function+ # TRL reloads the checkpoint, dropping in-memory training: copy the policy's weights into it.+ # Not deepcopy: instance forward patches (block-compiled experts, packing) close over the policy's modules.+ function, count = re.subn(+ r"(?m)^(?P<indent>[ \t]*)ref_model_init_kwargs = [^\n]+\n"+ r"(?:(?P=indent)[^\n]*\n)*?"+ r"(?P=indent)self\.ref_model = create_model_from_path\(ref_model_path, \*\*ref_model_init_kwargs\)\n",+ lambda m: m[0]+ + textwrap.indent(+ "from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled\n"+ "if _unsloth_dpo_model_is_instance and not is_deepspeed_zero3_enabled():\n"+ # Unbound calls: rl.py strips every literal `.state_dict()` (legacy vLLM weight loads).+ " _unsloth_policy_sd = torch.nn.Module.state_dict(self.model)\n"+ # Checked first: load_state_dict writes matching tensors before raising on the rest.+ " if _unsloth_dpo_reference_loadable(_unsloth_policy_sd, torch.nn.Module.state_dict(self.ref_model)):\n"+ " _unsloth_dpo_copy_policy_into_reference(self.ref_model, _unsloth_policy_sd)\n"+ " else:\n"+ # Aliased: a bare `import warnings` here would shadow TRL's module-level one in all of __init__.+ " import warnings as _unsloth_warnings\n"+ " _unsloth_warnings.warn(\n"+ ' "Unsloth: the in-memory policy\'s parameters do not match its checkpoint (resized, untied or "\n'+ ' "sharded), so the DPO reference uses the checkpoint weights. Pass ref_model= to choose it."\n'+ " )\n"+ " del _unsloth_policy_sd\n",+ m["indent"],+ ),+ function,+ count = 1,+ )+ if count:+ function = _insert_after_docstring(+ function, ["_unsloth_dpo_model_is_instance = not isinstance(model, str)"]+ )+ return function+++RL_FUNCTIONS["dpo_trainer"].append(dpo_trainer_reference_snapshot)+RL_PRE_ITEMS["dpo_trainer"].append(inspect.getsource(_unsloth_dpo_reference_loadable))+RL_PRE_ITEMS["dpo_trainer"].append(inspect.getsource(_unsloth_dpo_copy_policy_into_reference))RL_FUNCTIONS["dpo_trainer"].append(dpo_trainer_prepare_dataset)RL_PRE_ITEMS["dpo_trainer"].append(inspect.getsource(dpo_trainer_vision_process_row))RL_FUNCTIONS["dpo_trainer"].append(dpo_trainer_vision_signature_columns)