Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let peft 0.19 attach a LoRA to torchao int8 / fp8 weights of quantised single files (#17087)

Daniel Hanf46fbab62dd6
diff --git a/studio/backend/core/inference/diffusion_torchao_patches.py b/studio/backend/core/inference/diffusion_torchao_patches.py
index cc1e7a1ad8c..a8714ed0a47 100644
--- a/studio/backend/core/inference/diffusion_torchao_patches.py
+++ b/studio/backend/core/inference/diffusion_torchao_patches.py
@@ -356,6 +356,48 @@ def _guard_peft_torchao_dispatcher(original):
_PEFT_LORA_DISPATCH_MODULE = "peft.tuners.lora.model"
_PEFT_TORCHAO_GUARD_SENTINEL = "__unsloth_studio_peft_torchao_guard__"
+_PEFT_TORCHAO_LAYER_MODULE = "peft.tuners.lora.torchao"
+_PEFT_TORCHAO_MERGE_CALLBACK = "get_apply_tensor_subclass"
+# "0" keeps peft 0.19's required TorchaoLoraLinear(get_apply_tensor_subclass=...) keyword.
+_PEFT_TORCHAO_OPTIONAL_CALLBACK_ENV = "UNSLOTH_STUDIO_PEFT_TORCHAO_OPTIONAL_CALLBACK"
+
+
+def _make_peft_torchao_merge_callback_optional():
+ """peft 0.19 requires TorchaoLoraLinear's requantisation callback, found only on a transformers hf_quantizer, so a
+ LoRA on torchao weights built any other way raised at injection; backport peft 0.20's None default."""
+ if (os.environ.get(_PEFT_TORCHAO_OPTIONAL_CALLBACK_ENV) or "").strip().lower() in (
+ "0",
+ "off",
+ "false",
+ "no",
+ ):
+ return False
+ module = sys.modules.get(_PEFT_TORCHAO_LAYER_MODULE)
+ layer = getattr(module, "TorchaoLoraLinear", None) if module is not None else None
+ original = getattr(layer, "__init__", None) if isinstance(layer, type) else None
+ if original is None or getattr(original, "__unsloth_patched__", False):
+ return False
+ try:
+ parameter = inspect.signature(original).parameters.get(_PEFT_TORCHAO_MERGE_CALLBACK)
+ except (TypeError, ValueError):
+ return False
+ if (
+ parameter is None
+ or parameter.kind is not parameter.KEYWORD_ONLY
+ or parameter.default is not parameter.empty
+ ):
+ return (
+ False # peft <= 0.18 dispatches only its own torchao tensors; >= 0.20 has the default
+ )
+
+ @functools.wraps(original)
+ def __init__(self, *args, **kwargs):
+ kwargs.setdefault(_PEFT_TORCHAO_MERGE_CALLBACK, None)
+ original(self, *args, **kwargs)
+
+ __init__.__unsloth_patched__ = True
+ layer.__init__ = __init__
+ return True
def _patch_peft_torchao_dispatchers():
@@ -410,6 +452,10 @@ class _PeftLoraDispatchLoader(importlib.abc.Loader):
_patch_peft_torchao_dispatchers()
except Exception:
pass
+ try:
+ _make_peft_torchao_merge_callback_optional()
+ except Exception:
+ pass
def __getattr__(self, attribute):
return getattr(self._loader, attribute)
@@ -457,6 +503,10 @@ def install_peft_torchao_dispatch_guard():
except Exception:
return None
if _PEFT_LORA_DISPATCH_MODULE in sys.modules:
+ try:
+ _make_peft_torchao_merge_callback_optional()
+ except Exception:
+ pass
try:
return _patch_peft_torchao_dispatchers()
except Exception:
diff --git a/studio/backend/tests/test_diffusion_lora_quant_merge.py b/studio/backend/tests/test_diffusion_lora_quant_merge.py
index 32459887e1d..d746d0e6132 100644
--- a/studio/backend/tests/test_diffusion_lora_quant_merge.py
+++ b/studio/backend/tests/test_diffusion_lora_quant_merge.py
@@ -287,3 +287,38 @@ def test_a_failed_backup_allocation_keeps_the_adapter_layers(monkeypatch):
assert isinstance(net.proj["q"], lf.kron_linear_class()) and pipe.unloaded == 0
assert all(torch.equal(_codes(net)[k], original[k]) for k in original)
assert not qm.merged(pipe)
+
+
[email protected]("kind", ["int8", "fp8"])
+def test_a_peft_lora_injects_and_runs_on_a_quantized_single_file_weight(kind):
+ # diffusers' own loader on the torchao weights a quantised single file rebuilds (peft 0.19 raised a TypeError here)
+ pytest.importorskip("peft")
+ pytest.importorskip("diffusers")
+ from diffusers.loaders import PeftAdapterMixin
+
+ from core.inference.diffusion_torchao_patches import install_peft_torchao_dispatch_guard
+
+ install_peft_torchao_dispatch_guard()
+
+ class _Model(PeftAdapterMixin, _Net):
+ pass
+
+ net = _Model(kind)
+ weight = net.proj["q"].weight
+ x = torch.randn(3, 64, dtype = torch.bfloat16)
+ torch.manual_seed(2)
+ a, b = torch.randn(2, 64) * 0.1, torch.randn(32, 2) * 0.1
+ net.load_lora_adapter(
+ {"transformer.proj.q.lora_A.weight": a, "transformer.proj.q.lora_B.weight": b},
+ adapter_name = "l",
+ prefix = "transformer",
+ )
+ layer = net.proj["q"]
+ assert "l" in layer.lora_A and layer.get_base_layer().weight is weight
+ if kind == "fp8":
+ return # torch's CPU _scaled_mm takes per-tensor scales only; the fp8 forward runs on a GPU
+ with torch.no_grad():
+ got = layer(x).float()
+ base_out = layer.get_base_layer()(x).float()
+ want = base_out + (x.float() @ a.T.bfloat16().float() @ b.T.bfloat16().float())
+ assert (got - want).norm() / want.norm() < 0.02
diff --git a/studio/backend/tests/test_diffusion_peft_torchao_guard.py b/studio/backend/tests/test_diffusion_peft_torchao_guard.py
index dc8d65aa0eb..cced64c167b 100644
--- a/studio/backend/tests/test_diffusion_peft_torchao_guard.py
+++ b/studio/backend/tests/test_diffusion_peft_torchao_guard.py
@@ -239,3 +239,64 @@ def test_real_peft_lora_injection_survives_the_installed_torchao():
net = torch.nn.Sequential(torch.nn.Linear(8, 8))
peft.inject_adapter_in_model(peft.LoraConfig(r = 2, target_modules = ["0"]), net)
assert hasattr(net[0], "lora_A")
+
+
+def _fake_torchao_layer(monkeypatch, init):
+ module = types.ModuleType(patches._PEFT_TORCHAO_LAYER_MODULE)
+ module.TorchaoLoraLinear = type("TorchaoLoraLinear", (), {"__init__": init})
+ monkeypatch.setitem(sys.modules, patches._PEFT_TORCHAO_LAYER_MODULE, module)
+ monkeypatch.delenv(patches._PEFT_TORCHAO_OPTIONAL_CALLBACK_ENV, raising = False)
+ return module.TorchaoLoraLinear
+
+
+def test_peft_019s_required_merge_callback_gets_peft_020s_default(monkeypatch):
+ seen = []
+
+ def init(self, *args, get_apply_tensor_subclass, **kwargs): # peft 0.19.x
+ seen.append(get_apply_tensor_subclass)
+
+ layer = _fake_torchao_layer(monkeypatch, init)
+ with pytest.raises(TypeError, match = "get_apply_tensor_subclass"):
+ layer()
+ assert patches._make_peft_torchao_merge_callback_optional() is True
+ assert patches._make_peft_torchao_merge_callback_optional() is False
+ layer()
+ callback = object()
+ layer(get_apply_tensor_subclass = callback) # a transformers hf_quantizer's callback still wins
+ assert seen == [None, callback]
+ assert layer.__init__.__wrapped__ is init
+
+
+ "signature",
+ ["optional", "absent"], # peft >= 0.20 / a layer without the keyword: left alone
+)
+def test_other_peft_layers_are_left_alone(monkeypatch, signature):
+ if signature == "optional":
+
+ def init(
+ self,
+ *args,
+ get_apply_tensor_subclass = None,
+ **kwargs,
+ ):
+ pass
+
+ else:
+
+ def init(self, *args, **kwargs):
+ pass
+
+ layer = _fake_torchao_layer(monkeypatch, init)
+ assert patches._make_peft_torchao_merge_callback_optional() is False
+ assert layer.__init__ is init
+
+
+def test_the_merge_callback_switch_keeps_the_required_keyword(monkeypatch):
+ def init(self, *args, get_apply_tensor_subclass, **kwargs):
+ pass
+
+ layer = _fake_torchao_layer(monkeypatch, init)
+ monkeypatch.setenv(patches._PEFT_TORCHAO_OPTIONAL_CALLBACK_ENV, "0")
+ assert patches._make_peft_torchao_merge_callback_optional() is False
+ assert layer.__init__ is init