Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Tests: undo the Unsloth load's PEFT LoRA forward swap after the probe memo tests (#16908)

Daniel Han7a1325667ce0
diff --git a/tests/python/test_load_probe_memo.py b/tests/python/test_load_probe_memo.py
index 5d78b645bfc..6c92c7ed7d8 100644
--- a/tests/python/test_load_probe_memo.py
+++ b/tests/python/test_load_probe_memo.py
@@ -2,8 +2,11 @@
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
import json
+import os
+import sys
import pytest
+import torch
import unsloth
import peft
@@ -12,6 +15,36 @@ import transformers
from unsloth.models import loader_utils
+def _lora_forwards():
+ return {
+ cls: vars(cls)["forward"]
+ for name, module in list(sys.modules.items())
+ if name.startswith("peft.tuners.lora") and module is not None
+ for cls in vars(module).values()
+ if isinstance(cls, type)
+ and issubclass(cls, torch.nn.Module)
+ and cls.__module__.startswith("peft.")
+ and "forward" in vars(cls)
+ }
+
+
[email protected](autouse = True)
+def _undo_process_wide_load_patches():
+ """FastModel.from_pretrained swaps PEFT's LoRA forwards and sets UNSLOTH_* variables for the whole process. Put
+ both back, or a later test in the same xdist worker runs on Unsloth's LoRA forward without asking for it."""
+ env = {k: v for k, v in os.environ.items() if k.startswith("UNSLOTH_")}
+ forwards = _lora_forwards()
+ yield
+ for cls, forward in _lora_forwards().items():
+ # A PEFT module first imported during the load has no entry; fall back to the forward Unsloth wrapped.
+ original = forwards.get(cls, getattr(forward, "_unsloth_original_forward", forward))
+ if forward is not original:
+ cls.forward = original
+ for key in [k for k in os.environ if k.startswith("UNSLOTH_") and k not in env]:
+ del os.environ[key]
+ os.environ.update(env)
+
+
def _save_tiny(tmp_path, name, config):
path = tmp_path / name
config.save_pretrained(str(path))
diff --git a/tests/python/test_preference_chunked_logprobs.py b/tests/python/test_preference_chunked_logprobs.py
index 8afc042b064..07983e0ba93 100644
--- a/tests/python/test_preference_chunked_logprobs.py
+++ b/tests/python/test_preference_chunked_logprobs.py
@@ -8,6 +8,7 @@ import ast
import functools
import importlib.util
import os
+import sys
import textwrap
import types
@@ -146,6 +147,19 @@ class _ToyLM(torch.nn.Module):
return cap * torch.tanh(z / cap)
+def _pin_stock_peft_lora(monkeypatch):
+ """The reference is TRL on stock PEFT. An Unsloth load earlier in the worker swaps PEFT's LoRA forwards for the
+ whole process, which moves these gradients past the tolerance below."""
+ for name, module in list(sys.modules.items()):
+ if not name.startswith("peft.tuners.lora") or module is None:
+ continue
+ for cls in list(vars(module).values()):
+ forward = vars(cls).get("forward") if isinstance(cls, type) else None
+ original = getattr(forward, "_unsloth_original_forward", None)
+ if original is not None:
+ monkeypatch.setattr(cls, "forward", original)
+
+
def _qwen_lora():
transformers = pytest.importorskip("transformers")
peft = pytest.importorskip("peft")
@@ -250,6 +264,8 @@ def test_matches_trl(kind, kwargs, model_name, monkeypatch):
), "rewrite did not recognise the installed TRL concatenated_forward"
stock, patched = _compile(source), _compile(patched_source, rows_per_chunk = 5)
+ if model_name == "qwen3_lora":
+ _pin_stock_peft_lora(monkeypatch)
lm = _qwen_lora() if model_name == "qwen3_lora" else _ToyLM()
model = _HiddenStatesLM(lm)
params = [p for p in model.parameters() if p.requires_grad]