Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: let peft 0.19 attach a LoRA to torchao int8 / fp8 weights of quantised single files (#17087)
Daniel Hanf46fbab62dd6
diff --git a/studio/backend/core/inference/diffusion_torchao_patches.py b/studio/backend/core/inference/diffusion_torchao_patches.pyindex cc1e7a1ad8c..a8714ed0a47 100644--- a/studio/backend/core/inference/diffusion_torchao_patches.py+++ b/studio/backend/core/inference/diffusion_torchao_patches.py@@ -356,6 +356,48 @@ def _guard_peft_torchao_dispatcher(original):_PEFT_LORA_DISPATCH_MODULE = "peft.tuners.lora.model"_PEFT_TORCHAO_GUARD_SENTINEL = "__unsloth_studio_peft_torchao_guard__"+_PEFT_TORCHAO_LAYER_MODULE = "peft.tuners.lora.torchao"+_PEFT_TORCHAO_MERGE_CALLBACK = "get_apply_tensor_subclass"+# "0" keeps peft 0.19's required TorchaoLoraLinear(get_apply_tensor_subclass=...) keyword.+_PEFT_TORCHAO_OPTIONAL_CALLBACK_ENV = "UNSLOTH_STUDIO_PEFT_TORCHAO_OPTIONAL_CALLBACK"+++def _make_peft_torchao_merge_callback_optional():+ """peft 0.19 requires TorchaoLoraLinear's requantisation callback, found only on a transformers hf_quantizer, so a+ LoRA on torchao weights built any other way raised at injection; backport peft 0.20's None default."""+ if (os.environ.get(_PEFT_TORCHAO_OPTIONAL_CALLBACK_ENV) or "").strip().lower() in (+ "0",+ "off",+ "false",+ "no",+ ):+ return False+ module = sys.modules.get(_PEFT_TORCHAO_LAYER_MODULE)+ layer = getattr(module, "TorchaoLoraLinear", None) if module is not None else None+ original = getattr(layer, "__init__", None) if isinstance(layer, type) else None+ if original is None or getattr(original, "__unsloth_patched__", False):+ return False+ try:+ parameter = inspect.signature(original).parameters.get(_PEFT_TORCHAO_MERGE_CALLBACK)+ except (TypeError, ValueError):+ return False+ if (+ parameter is None+ or parameter.kind is not parameter.KEYWORD_ONLY+ or parameter.default is not parameter.empty+ ):+ return (+ False # peft <= 0.18 dispatches only its own torchao tensors; >= 0.20 has the default+ )++ @functools.wraps(original)+ def __init__(self, *args, **kwargs):+ kwargs.setdefault(_PEFT_TORCHAO_MERGE_CALLBACK, None)+ original(self, *args, **kwargs)++ __init__.__unsloth_patched__ = True+ layer.__init__ = __init__+ return Truedef _patch_peft_torchao_dispatchers():@@ -410,6 +452,10 @@ class _PeftLoraDispatchLoader(importlib.abc.Loader):_patch_peft_torchao_dispatchers()except Exception:pass+ try:+ _make_peft_torchao_merge_callback_optional()+ except Exception:+ passdef __getattr__(self, attribute):return getattr(self._loader, attribute)@@ -457,6 +503,10 @@ def install_peft_torchao_dispatch_guard():except Exception:return Noneif _PEFT_LORA_DISPATCH_MODULE in sys.modules:+ try:+ _make_peft_torchao_merge_callback_optional()+ except Exception:+ passtry:return _patch_peft_torchao_dispatchers()except Exception:diff --git a/studio/backend/tests/test_diffusion_lora_quant_merge.py b/studio/backend/tests/test_diffusion_lora_quant_merge.pyindex 32459887e1d..d746d0e6132 100644--- a/studio/backend/tests/test_diffusion_lora_quant_merge.py+++ b/studio/backend/tests/test_diffusion_lora_quant_merge.py@@ -287,3 +287,38 @@ def test_a_failed_backup_allocation_keeps_the_adapter_layers(monkeypatch):assert isinstance(net.proj["q"], lf.kron_linear_class()) and pipe.unloaded == 0assert all(torch.equal(_codes(net)[k], original[k]) for k in original)assert not qm.merged(pipe)++[email protected]("kind", ["int8", "fp8"])+def test_a_peft_lora_injects_and_runs_on_a_quantized_single_file_weight(kind):+ # diffusers' own loader on the torchao weights a quantised single file rebuilds (peft 0.19 raised a TypeError here)+ pytest.importorskip("peft")+ pytest.importorskip("diffusers")+ from diffusers.loaders import PeftAdapterMixin++ from core.inference.diffusion_torchao_patches import install_peft_torchao_dispatch_guard++ install_peft_torchao_dispatch_guard()++ class _Model(PeftAdapterMixin, _Net):+ pass++ net = _Model(kind)+ weight = net.proj["q"].weight+ x = torch.randn(3, 64, dtype = torch.bfloat16)+ torch.manual_seed(2)+ a, b = torch.randn(2, 64) * 0.1, torch.randn(32, 2) * 0.1+ net.load_lora_adapter(+ {"transformer.proj.q.lora_A.weight": a, "transformer.proj.q.lora_B.weight": b},+ adapter_name = "l",+ prefix = "transformer",+ )+ layer = net.proj["q"]+ assert "l" in layer.lora_A and layer.get_base_layer().weight is weight+ if kind == "fp8":+ return # torch's CPU _scaled_mm takes per-tensor scales only; the fp8 forward runs on a GPU+ with torch.no_grad():+ got = layer(x).float()+ base_out = layer.get_base_layer()(x).float()+ want = base_out + (x.float() @ a.T.bfloat16().float() @ b.T.bfloat16().float())+ assert (got - want).norm() / want.norm() < 0.02diff --git a/studio/backend/tests/test_diffusion_peft_torchao_guard.py b/studio/backend/tests/test_diffusion_peft_torchao_guard.pyindex dc8d65aa0eb..cced64c167b 100644--- a/studio/backend/tests/test_diffusion_peft_torchao_guard.py+++ b/studio/backend/tests/test_diffusion_peft_torchao_guard.py@@ -239,3 +239,64 @@ def test_real_peft_lora_injection_survives_the_installed_torchao():net = torch.nn.Sequential(torch.nn.Linear(8, 8))peft.inject_adapter_in_model(peft.LoraConfig(r = 2, target_modules = ["0"]), net)assert hasattr(net[0], "lora_A")+++def _fake_torchao_layer(monkeypatch, init):+ module = types.ModuleType(patches._PEFT_TORCHAO_LAYER_MODULE)+ module.TorchaoLoraLinear = type("TorchaoLoraLinear", (), {"__init__": init})+ monkeypatch.setitem(sys.modules, patches._PEFT_TORCHAO_LAYER_MODULE, module)+ monkeypatch.delenv(patches._PEFT_TORCHAO_OPTIONAL_CALLBACK_ENV, raising = False)+ return module.TorchaoLoraLinear+++def test_peft_019s_required_merge_callback_gets_peft_020s_default(monkeypatch):+ seen = []++ def init(self, *args, get_apply_tensor_subclass, **kwargs): # peft 0.19.x+ seen.append(get_apply_tensor_subclass)++ layer = _fake_torchao_layer(monkeypatch, init)+ with pytest.raises(TypeError, match = "get_apply_tensor_subclass"):+ layer()+ assert patches._make_peft_torchao_merge_callback_optional() is True+ assert patches._make_peft_torchao_merge_callback_optional() is False+ layer()+ callback = object()+ layer(get_apply_tensor_subclass = callback) # a transformers hf_quantizer's callback still wins+ assert seen == [None, callback]+ assert layer.__init__.__wrapped__ is init+++ "signature",+ ["optional", "absent"], # peft >= 0.20 / a layer without the keyword: left alone+)+def test_other_peft_layers_are_left_alone(monkeypatch, signature):+ if signature == "optional":++ def init(+ self,+ *args,+ get_apply_tensor_subclass = None,+ **kwargs,+ ):+ pass++ else:++ def init(self, *args, **kwargs):+ pass++ layer = _fake_torchao_layer(monkeypatch, init)+ assert patches._make_peft_torchao_merge_callback_optional() is False+ assert layer.__init__ is init+++def test_the_merge_callback_switch_keeps_the_required_keyword(monkeypatch):+ def init(self, *args, get_apply_tensor_subclass, **kwargs):+ pass++ layer = _fake_torchao_layer(monkeypatch, init)+ monkeypatch.setenv(patches._PEFT_TORCHAO_OPTIONAL_CALLBACK_ENV, "0")+ assert patches._make_peft_torchao_merge_callback_optional() is False+ assert layer.__init__ is init