Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: apply ComfyUI LoRA and LoKR files to Anima, Krea 2 and MOSS-Image (#16711)
oobabooga083ba2f8c122
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex a045e158cfd..7ff5c699ab5 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -393,6 +393,7 @@ from .diffusion_precision import (torchao_quantize_importable,)from .diffusion_te_prequant import supplied_component_pipe_kwargs, te_prequant_pipe_kwargs+from . import diffusion_lora_files as lora_filesfrom .diffusion_fast_load import _snapshot_dir as _load_snapshot_dirfrom .diffusion_fast_load import start_load_prefetch, stop_prefetch, te_precast_componentsfrom .diffusion_hosted_vae import hosted_vae_pipe_kwargs, hosted_vae_plan_files@@ -897,6 +898,8 @@ def _baked_lora_names(pipe: Any) -> list:return names+# Adapters baked into a quantized single file; ``_unsloth_loras_baked`` also stays set after a declined quantize.+_QUANT_FILE_LORA_BAKE_ATTR = "_unsloth_quant_file_loras_baked"# ``(module, adapter, base)`` for every baked LoRA scale held as a device tensor; ``base`` is lora_alpha / r._PINNED_LORA_SCALES_ATTR = "_unsloth_pinned_lora_scales"# ``(module, adapter)`` for the baked LoRA layers OUTSIDE the denoisers (a text encoder carrying the same file's@@ -994,9 +997,16 @@ def _set_pinned_lora_scales(pipe: Any, weights: dict) -> bool:weights): the caller then takes the set_adapters + graph reset path."""pinned = getattr(pipe, _PINNED_LORA_SCALES_ATTR, None)if not pinned:+ if lora_files.kron_only_selection(pipe):+ # LoKR scales are tensors written in place; the CFG replica holds copies, so it is rebuilt.+ lora_files.set_kron_weights(pipe, weights)+ invalidate_cfg_replica(pipe, "the LoKR weights changed")+ return Truereturn Falseimport torch+ lora_files.set_kron_weights(pipe, weights)+for module, adapter, _base in pinned:value = module.scaling.get(adapter)if (@@ -10267,10 +10277,13 @@ class DiffusionBackend:hf_token = hf_token,)for name, path, _weight in baked:- pipe.load_lora_weights(path, adapter_name = name)- pipe.set_adapters(+ lora_files.load_adapter(+ pipe, path, name, getattr(fam, "name", None)+ )+ lora_files.set_adapter_weights(+ pipe,[n for (n, _p, _w) in baked],- adapter_weights = [w for (_n, _p, w) in baked],+ [w for (_n, _p, w) in baked],)pipe._unsloth_loras = bakedpipe._unsloth_loras_baked = True@@ -10409,6 +10422,28 @@ class DiffusionBackend:effective_speed = resolve_speed_mode(speed_mode, is_gguf = kind == "gguf")split_expected = _layer_split_expected(pipe, plan)# A torchao (or torchao-free resident) int8 DiT must be compiled: eager is ~30x slower, losing to GGUF.+ if comfy_compile and _has_active_lora(loras):+ # A quantized single file bakes its adapters before the compile; a new adapter set reloads.+ baked = self._resolve_lora_set(+ [(i, w) for (i, w) in loras if w != 0],+ family = getattr(fam, "name", None),+ hf_token = hf_token,+ )+ for name, path, _weight in baked:+ lora_files.load_adapter(pipe, path, name, getattr(fam, "name", None))+ lora_files.set_adapter_weights(+ pipe,+ [n for (n, _p, _w) in baked],+ [w for (_n, _p, w) in baked],+ )+ pipe._unsloth_loras = baked+ pipe._unsloth_loras_baked = True+ setattr(pipe, _QUANT_FILE_LORA_BAKE_ATTR, True)+ logger.info(+ "diffusion.lora_bake: %d adapter(s) attached to the ComfyUI-quantized transformer "+ "before the compile",+ len(baked),+ )quant_forces_compile = (transformer_quant_engaged is not None or comfy_compile) and (@@ -11684,11 +11719,12 @@ class DiffusionBackend:)for name, path, _weight in baked:check_cancelled()- pipe.load_lora_weights(path, adapter_name = name)+ lora_files.load_adapter(pipe, path, name, getattr(fam, "name", None))check_cancelled()- pipe.set_adapters(+ lora_files.set_adapter_weights(+ pipe,[n for (n, _p, _w) in baked],- adapter_weights = [w for (_n, _p, w) in baked],+ [w for (_n, _p, w) in baked],)pipe._unsloth_loras = bakedpipe._unsloth_loras_baked = True@@ -13522,7 +13558,10 @@ class DiffusionBackend:_refuse_lora_without_loader(getattr(state, "family", None), specs)quant_baked = bool(getattr(pipe, "_unsloth_loras_baked", False))quant = (state.transformer_quant or "").lower()- if quant in ("int8", "fp8", "nvfp4", "mxfp8"):+ # A quantized single file reports no transformer_quant but bakes its adapters the same way.+ if quant in ("int8", "fp8", "nvfp4", "mxfp8") or getattr(+ pipe, _QUANT_FILE_LORA_BAKE_ATTR, False+ ):self._adjust_baked_loras(state, pipe, specs, current, quant_baked, cancel)return@@ -13530,7 +13569,7 @@ class DiffusionBackend:if current:self._unfuse_loras(state, pipe)try:- pipe.unload_lora_weights()+ lora_files.unload_adapters(pipe)except Exception: # noqa: BLE001 -- best-effort clearpasspipe._unsloth_loras = ()@@ -13568,15 +13607,17 @@ class DiffusionBackend:weights = [w for _n, _p, w in desired]if _lora_weights_only_change(state, pipe, current, desired):try:- pipe.set_adapters(names, adapter_weights = weights)+ lora_files.set_adapter_weights(pipe, names, weights)except Exception: # noqa: BLE001 -- the full reload below sets the same scales or fails cleanlypasselse:pipe._unsloth_loras = desired- # A captured graph bakes the scales in.- cuda_graph.reset_all(getattr(state, "cuda_graphs", ()))- # Same adapters, new scales: the replica takes the scales; a rebuild only when they do not line up.- if not mirror_cfg_replica_adapter_scales(pipe, logger):+ # A captured graph bakes PEFT's float scales in; LoKR scales are tensors it reads at run time.+ if not lora_files.kron_only_selection(pipe):+ cuda_graph.reset_all(getattr(state, "cuda_graphs", ()))+ # The scale mirror skips LoKR buffers: LoKR or misaligned layers rebuild the replica.+ kron = lora_files.kron_names(pipe)+ if kron or not mirror_cfg_replica_adapter_scales(pipe, logger):invalidate_cfg_replica(pipe, "the LoRA adapters changed", logger)return# PEFT targets to_q / to_k / to_v: a fused U-Net that dropped them gets them back first (bit-exact).@@ -13585,14 +13626,14 @@ class DiffusionBackend:file_sigs = _lora_file_sigs(desired)try:if current:- pipe.unload_lora_weights()+ lora_files.unload_adapters(pipe)setattr(pipe, _LORA_FILE_SIGS_ATTR, None)for name, path, _weight in desired:- pipe.load_lora_weights(path, adapter_name = name)- pipe.set_adapters(names, adapter_weights = weights)+ lora_files.load_adapter(pipe, path, name, getattr(state.family, "name", None))+ lora_files.set_adapter_weights(pipe, names, weights)except Exception as exc: # noqa: BLE001 -- surface as a clean 400try:- pipe.unload_lora_weights()+ lora_files.unload_adapters(pipe)except Exception: # noqa: BLE001passpipe._unsloth_loras = ()@@ -13664,7 +13705,7 @@ class DiffusionBackend:if _set_pinned_lora_scales(pipe, {}):pipe._unsloth_loras = disabledreturn- pipe.set_adapters(names, adapter_weights = [0.0] * len(names))+ lora_files.set_adapter_weights(pipe, names, [0.0] * len(names))pipe._unsloth_loras = disabled# Adapter scales are Python values read inside the LoRA forward, so a graph bakes them in.cuda_graph.reset_all(getattr(state, "cuda_graphs", ()))@@ -13696,9 +13737,10 @@ class DiffusionBackend:if _set_pinned_lora_scales(pipe, {n: w for (n, _p, w) in updated}):pipe._unsloth_loras = updatedreturn- pipe.set_adapters(+ lora_files.set_adapter_weights(+ pipe,[n for (n, _p, _w) in updated],- adapter_weights = [w for (_n, _p, w) in updated],+ [w for (_n, _p, w) in updated],)pipe._unsloth_loras = updatedcuda_graph.reset_all(getattr(state, "cuda_graphs", ()))@@ -15626,7 +15668,10 @@ class DiffusionBackend:model_kind = state.kind,transformer_quant = state.transformer_quant,)- compiled = "compiled" in (getattr(state, "speed_optims", ()) or ())+ # Baked adapters ride inside the compiled build.+ compiled = "compiled" in (getattr(state, "speed_optims", ()) or ()) and not getattr(+ state.pipe, _QUANT_FILE_LORA_BAKE_ATTR, False+ )lora_block = _lora_load_block_reason(state.pipe)lora_load_ok = lora_block is Nonelora_reason = lora_block or diffusion_lora.lora_unavailable_reason(diff --git a/studio/backend/core/inference/diffusion_lora.py b/studio/backend/core/inference/diffusion_lora.pyindex 57a0b35cb65..a036e4b6651 100644--- a/studio/backend/core/inference/diffusion_lora.py+++ b/studio/backend/core/inference/diffusion_lora.py@@ -670,7 +670,7 @@ _DIFFUSERS_LORA_BAKED_QUANT = ("int8", "fp8")# Prototype schemes with no validated LoRA path (and no shipped families needing one)._DIFFUSERS_LORA_BLOCKED_QUANT = ("nvfp4", "mxfp8")# In-tree pipelines with no LoRA loader mixin (no load_lora_weights / set_adapters / unload_lora_weights).-_DIFFUSERS_LORA_UNSUPPORTED_FAMILIES = ("moss-image",)+_DIFFUSERS_LORA_UNSUPPORTED_FAMILIES: tuple[str, ...] = ()def diffusers_lora_family_supported(family: Optional[str]) -> bool:diff --git a/studio/backend/core/inference/diffusion_lora_files.py b/studio/backend/core/inference/diffusion_lora_files.pynew file mode 100644index 00000000000..30319586442--- /dev/null+++ b/studio/backend/core/inference/diffusion_lora_files.py@@ -0,0 +1,510 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""LoRA and LyCORIS LoKR files on Anima, Krea 2 and MOSS-Image. LoKR runs as ``KronLinear`` slots sharing PEFT's+adapter names: :func:`set_adapter_weights` / :func:`unload_adapters` replace diffusers' adapter calls."""++from __future__ import annotations++import re+from typing import Any, Optional++from loggers import get_logger++logger = get_logger(__name__)++KRON_ATTR = "_unsloth_kron_adapters"+# Adapter names whose file held LoKR modules only: diffusers has no PEFT adapter of that name to set.+KRON_ONLY_ATTR = "_unsloth_kron_only"+_PREFIXES = ("diffusion_model.", "model.diffusion_model.", "transformer.")+# Kohya / SimpleTuner keys: the reference layer path with its dots written as underscores.+_KOHYA_PREFIXES = ("lora_unet_", "lycoris_")+_LOKR_SUFFIXES = (+ "lokr_w1",+ "lokr_w2",+ "lokr_w1_a",+ "lokr_w1_b",+ "lokr_w2_a",+ "lokr_w2_b",+ "lokr_t2",+)+NORMALIZED_FAMILIES = ("anima", "krea-2", "moss-image")+++def read_lora_file(path: str) -> dict:+ from safetensors.torch import load_file+ return load_file(str(path))+++def _strip(key: str) -> str:+ for prefix in _PREFIXES:+ if key.startswith(prefix):+ return key[len(prefix) :]+ return key+++def _lokr_stem(key: str) -> Optional[str]:+ for suffix in _LOKR_SUFFIXES:+ if key.endswith("." + suffix):+ return key[: -len(suffix) - 1]+ return None+++def split_lokr(state: dict) -> tuple[dict, dict]:+ """``(lokr, rest)``: LoKR tensors by module stem (each with its ``.alpha``), and every other tensor."""+ lokr: dict[str, dict] = {}+ for key, value in state.items():+ stem = _lokr_stem(key)+ if stem is not None:+ lokr.setdefault(stem, {})[key[len(stem) + 1 :]] = value+ rest = {}+ for key, value in state.items():+ if _lokr_stem(key) is not None:+ continue+ stem, _, suffix = key.rpartition(".")+ if suffix in ("alpha", "dora_scale") and stem in lokr:+ lokr[stem][suffix] = value+ continue+ rest[key] = value+ return lokr, rest+++def normalize_lora(state: dict, family: Optional[str]) -> dict:+ """Plain LoRA tensors as ``lora_A`` / ``lora_B`` with ``.alpha`` folded into ``lora_B`` (alpha / rank)."""+ import torch++ renamed: dict = {}+ alphas: dict = {}+ for key, value in state.items():+ if key.endswith(".alpha"):+ alphas[key[: -len(".alpha")]] = value+ continue+ key = key.replace(".lora_down.", ".lora_A.").replace(".lora_up.", ".lora_B.")+ renamed[key] = value+ for stem, alpha in alphas.items():+ a, b = renamed.get(stem + ".lora_A.weight"), renamed.get(stem + ".lora_B.weight")+ if a is None or b is None:+ continue # an alpha with no low-rank pair (some converters write it for every module)+ scale = float(alpha) / int(a.shape[0])+ if scale != 1.0:+ renamed[stem + ".lora_B.weight"] = (b.to(torch.float32) * scale).to(b.dtype)+ if family == "anima":+ renamed = _anima_embedding(renamed)+ elif family == "moss-image":+ renamed = _moss_fused_qkv(renamed)+ return renamed+++def _anima_embedding(state: dict) -> dict:+ """An ``llm_adapter.embed`` pair is a Linear delta ``B @ A`` on the ``[vocab, dim]`` embedding+ table; PEFT's embedding LoRA computes ``(B' @ A').T``, so ``A' = B.T`` and ``B' = A.T``."""+ out = dict(state)+ for key in [k for k in state if ".llm_adapter.embed.lora_A.weight" in k]:+ stem = key[: -len(".lora_A.weight")]+ a, b = out.pop(stem + ".lora_A.weight"), out.pop(stem + ".lora_B.weight", None)+ if b is None:+ continue+ out[stem + ".lora_embedding_A"] = b.T.contiguous()+ out[stem + ".lora_embedding_B"] = a.T.contiguous()+ return out+++_MOSS_QKV = re.compile(r"^(.*\.proj\.)qkv\.(lora_[AB])\.weight$")+++def _moss_fused_qkv(state: dict) -> dict:+ """The fused ``proj.qkv`` as the in-tree ``proj.q`` / ``k`` / ``v``: ``A`` is shared, ``B``'s rows split."""+ out: dict = {}+ for key, value in state.items():+ match = _MOSS_QKV.match(key)+ if match is None:+ out[key] = value+ continue+ stem, part = match.groups()+ if part == "lora_A":+ for name in ("q", "k", "v"):+ out[f"{stem}{name}.lora_A.weight"] = value+ else:+ rows = int(value.shape[0]) // 3+ for i, name in enumerate(("q", "k", "v")):+ out[f"{stem}{name}.lora_B.weight"] = value[i * rows : (i + 1) * rows].contiguous()+ return out+++def _kron_delta(x: Any, w1: Any, w2: Any, rows: Optional[tuple[int, int]]) -> Any:+ """``x @ kron(w1, w2)[rows].T`` without the dense product: row ``i * b1 + k`` and column ``j * b2 + l`` of the+ Kronecker product is ``w1[i, j] * w2[k, l]``, so two small matmuls on a reshaped input give it."""+ a1, a2 = w1.shape+ b1, b2 = w2.shape+ lead = x.shape[:-1]+ t = x.reshape(*lead, a2, b2) @ w2.T # (..., a2, b1)+ y = (w1 @ t).reshape(*lead, a1 * b1) # (..., a1, b1) -> rows i * b1 + k+ return y if rows is None else y[..., rows[0] : rows[1]]+++def _module_class():+ from torch import nn+ class KronLinear(nn.Module):+ """A Linear plus LoKR deltas, a module rather than a hook (fused int8 paths and compiled forwards skip hooks).+ Each 0-d ``scale`` buffer is filled in place, so compiled code and CUDA graphs need no retrace."""++ def __init__(self, base_layer):+ super().__init__()+ self.base_layer = base_layer+ self.slots: list = []++ @property+ def weight(self):+ return self.base_layer.weight++ @property+ def bias(self):+ return self.base_layer.bias++ def get_base_layer(self):+ base = self.base_layer+ get = getattr(base, "get_base_layer", None)+ return get() if callable(get) else base++ def forward(self, x, *args, **kwargs):+ out = self.base_layer(x, *args, **kwargs)+ for prefix, rows in self.slots:+ w1 = getattr(self, prefix + "_w1").to(x.dtype)+ w2 = getattr(self, prefix + "_w2").to(x.dtype)+ out = out + _kron_delta(x, w1, w2, rows) * getattr(self, prefix + "_scale").to(+ out.dtype+ )+ return out++ return KronLinear+++_KRON_CLASS = None+++def kron_linear_class():+ global _KRON_CLASS+ if _KRON_CLASS is None:+ _KRON_CLASS = _module_class()+ return _KRON_CLASS+++def _parent_of(root: Any, module: Any) -> tuple[Any, str]:+ for name, child in root.named_modules():+ for attr, sub in child.named_children():+ if sub is module:+ return child, attr+ raise ValueError("the LoKR target is not a submodule of its component")+++class KronAdapter:+ def __init__(+ self,+ root: Any,+ module: Any,+ w1: Any,+ w2: Any,+ alpha_scale: float,+ rows: Optional[tuple[int, int]] = None,+ ):+ import torch++ cls = kron_linear_class()+ base = _base_layer(module)+ device = _device_of(base)+ dtype = _dtype_of(base)+ self.alpha_scale = float(alpha_scale)+ # Allocated before the layer is swapped, so a failure (OOM) leaves the model as it was.+ buffers = (+ w1.to(device = device, dtype = dtype),+ w2.to(device = device, dtype = dtype),+ torch.tensor(self.alpha_scale, dtype = torch.float32, device = device),+ )+ if not isinstance(module, cls):+ parent, attr = _parent_of(root, module)+ wrapper = cls(module)+ setattr(parent, attr, wrapper)+ module = wrapper+ self.wrapper = module+ self.root = root+ self.prefix = f"kron{len(module.slots)}"+ for suffix, tensor in zip(("_w1", "_w2", "_scale"), buffers):+ module.register_buffer(self.prefix + suffix, tensor, persistent = False)+ module.slots.append((self.prefix, rows))++ def set_weight(self, weight: float) -> None:+ import torch+ with torch.no_grad():+ getattr(self.wrapper, self.prefix + "_scale").fill_(float(weight) * self.alpha_scale)++ def remove(self) -> None:+ wrapper = self.wrapper+ wrapper.slots = [slot for slot in wrapper.slots if slot[0] != self.prefix]+ for suffix in ("_w1", "_w2", "_scale"):+ wrapper._buffers.pop(self.prefix + suffix, None)+ if not wrapper.slots:+ parent, attr = _parent_of(self.root, wrapper)+ setattr(parent, attr, wrapper.base_layer)+++def _kron_factors(parts: dict) -> tuple[Any, Any, float]:+ """``(w1, w2, alpha_scale)`` like the reference runtime: a factor stored as a low-rank pair is its product, and+ ``alpha / rank`` applies only when one is (rank = that pair's inner size)."""+ import torch++ dim = None+ w1 = parts.get("lokr_w1")+ if w1 is None:+ w1 = parts["lokr_w1_a"].float() @ parts["lokr_w1_b"].float()+ dim = int(parts["lokr_w1_b"].shape[0])+ w2 = parts.get("lokr_w2")+ if w2 is None:+ if "lokr_t2" in parts:+ raise ValueError("a LoKR convolution (lokr_t2) cannot apply to a Linear layer")+ w2 = parts["lokr_w2_a"].float() @ parts["lokr_w2_b"].float()+ dim = int(parts["lokr_w2_b"].shape[0])+ if w1.dim() != 2 or w2.dim() != 2:+ raise ValueError("only Linear LoKR modules are supported")+ if "dora_scale" in parts:+ raise ValueError("LoKR modules trained with DoRA (dora_scale) are not supported")+ alpha = parts.get("alpha")+ scale = float(alpha) / dim if alpha is not None and dim else 1.0+ return w1.to(torch.float32), w2.to(torch.float32), scale+++def _device_of(module: Any) -> Any:+ weight = getattr(module, "weight", None)+ return weight.device if weight is not None and weight.device.type != "meta" else None+++def _dtype_of(module: Any) -> Any:+ """The compute dtype of a layer: a quantized weight reports its storage dtype, so fall back to its bias or bf16."""+ import torch++ for name in ("weight", "bias"):+ tensor = getattr(module, name, None)+ if tensor is not None and tensor.is_floating_point() and tensor.element_size() >= 2:+ return tensor.dtype+ return torch.bfloat16+++def _base_layer(module: Any) -> Any:+ get_base = getattr(module, "get_base_layer", None)+ return get_base() if callable(get_base) else module+++def _lokr_targets(+ pipe: Any, stem: str, family: Optional[str]+) -> list[tuple[Any, Any, Optional[tuple[int, int]]]]:+ key = _strip(stem)+ if family == "krea-2":+ from .diffusion_single_file_converters import krea2_checkpoint_to_diffusers+ name = next(iter(krea2_checkpoint_to_diffusers({key + ".weight": None})))[: -len(".weight")]+ return [(pipe.transformer, pipe.transformer.get_submodule(name), None)]+ if family == "moss-image":+ root = pipe.transformer+ match = re.match(r"^(.*\.proj\.)qkv$", key)+ if match is None:+ return [(root, root.get_submodule(key), None)]+ modules = [root.get_submodule(match.group(1) + n) for n in ("q", "k", "v")]+ width = int(_base_layer(modules[0]).weight.shape[0])+ return [(root, m, (i * width, (i + 1) * width)) for i, m in enumerate(modules)]+ if family == "anima":+ from diffusers.loaders.lora_conversion_utils import (+ _convert_non_diffusers_anima_lora_to_diffusers,+ )++ name = next(+ iter(+ _convert_non_diffusers_anima_lora_to_diffusers(+ {"diffusion_model." + key + ".x": None}+ )+ )+ )+ component, _, path = name[: -len(".x")].partition(".")+ root = getattr(pipe, component)+ return [(root, root.get_submodule(path), None)]+ raise ValueError(f"LoKR adapters are not supported for {family or 'this model'}")+++def _attach_lokr(pipe: Any, name: str, lokr: dict, family: Optional[str]) -> int:+ from torch import nn++ adapters: list[KronAdapter] = []+ try:+ for stem, parts in lokr.items():+ w1, w2, scale = _kron_factors(parts)+ for root, module, rows in _lokr_targets(pipe, stem, family):+ base = _base_layer(module)+ if isinstance(base, nn.Embedding): # 2-D like a Linear, but its input is token ids+ raise ValueError(+ f"LoKR module {_strip(stem)} targets an embedding table; only Linear LoKR modules are supported"+ )+ out_f, in_f = (+ (int(d) for d in base.weight.shape) if base.weight.dim() == 2 else (0, 0)+ )+ full = int(w1.shape[0] * w2.shape[0])+ width = rows[1] - rows[0] if rows else full+ if in_f != int(w1.shape[1] * w2.shape[1]) or out_f != width:+ raise ValueError(+ f"LoKR module {_strip(stem)} is {full} x {int(w1.shape[1] * w2.shape[1])}, "+ f"but the layer it targets is {out_f} x {in_f}"+ )+ adapters.append(KronAdapter(root, module, w1, w2, scale, rows))+ except Exception:+ for adapter in adapters:+ adapter.remove()+ raise+ registry = pipe.__dict__.setdefault(KRON_ATTR, {})+ registry[name] = adapters+ return len(adapters)+++def _resolve_kohya_stem(pipe: Any, stem: str, family: Optional[str]) -> str:+ """The one dotted path, of every way to re-split ``stem``'s underscores, that names a layer of this model."""+ tokens = stem.split("_")+ if len(tokens) > 16:+ raise ValueError(f"LoRA key {stem} names no layer of this model")+ found = []+ for mask in range(1 << (len(tokens) - 1)):+ path = tokens[0] + "".join(+ ("." if mask >> i & 1 else "_") + token for i, token in enumerate(tokens[1:])+ )+ try:+ _lokr_targets(pipe, path, family)+ except Exception: # noqa: BLE001 -- this split names no layer+ continue+ found.append(path)+ if len(found) != 1:+ raise ValueError(f"LoRA key {stem} names no single layer of this model")+ return found[0]+++def _dotted_kohya_keys(pipe: Any, state: dict, family: Optional[str]) -> dict:+ out: dict = {}+ paths: dict = {}+ for key, value in state.items():+ prefix = next((p for p in _KOHYA_PREFIXES if key.startswith(p)), None)+ if prefix is None:+ out[key] = value+ continue+ stem, _, suffix = key[len(prefix) :].partition(".")+ if stem not in paths:+ paths[stem] = _resolve_kohya_stem(pipe, stem, family)+ out[f"diffusion_model.{paths[stem]}.{suffix}"] = value+ return out+++def _adapter_loaded(pipe: Any, name: str) -> bool:+ return any(name in names for names in (pipe.get_list_adapters() or {}).values())+++def load_adapter(pipe: Any, path: str, name: str, family: Optional[str]) -> None:+ """One file under ``name``: plain LoRA via ``pipe.load_lora_weights``, LoKR modules as ``KronLinear`` slots."""+ if family not in NORMALIZED_FAMILIES:+ pipe.load_lora_weights(path, adapter_name = name)+ return+ raw = read_lora_file(path)+ lokr, rest = split_lokr(_dotted_kohya_keys(pipe, raw, family))+ plain = normalize_lora(rest, family)+ if not any(".lora_" in k for k in plain):+ plain = {}+ if plain:+ # A file needing no rewrite loads by path: diffusers reads lora_alpha / rank from its metadata, not a dict.+ unchanged = not lokr and plain.keys() == raw.keys() and all(plain[k] is raw[k] for k in raw)+ # PEFT injects only into bare Linears: unwrap LoKR while it runs, then rewrap whatever sits there now.+ wrappers = _unwrap_kron(pipe)+ try:+ pipe.load_lora_weights(path if unchanged else plain, adapter_name = name)+ finally:+ _rewrap_kron(wrappers)+ if wrappers:+ _refresh_group_offload(_kron_roots(pipe))+ # diffusers drops keys outside the components it loads into without an error.+ if not _adapter_loaded(pipe, name):+ raise ValueError(f"{name}: none of its LoRA keys name a layer of this model")+ if lokr:+ attached = _attach_lokr(pipe, name, lokr, family)+ _refresh_group_offload(_kron_roots(pipe))+ if not plain:+ pipe.__dict__.setdefault(KRON_ONLY_ATTR, set()).add(name)+ logger.info("diffusion.lora: %s applies %d LoKR module(s)", name, attached)+ if not plain and not lokr:+ raise ValueError(f"{name} holds no LoRA or LoKR weights this loader recognises")+++def _kron_roots(pipe: Any) -> list:+ roots = {+ id(adapter.root): adapter.root+ for adapters in pipe.__dict__.get(KRON_ATTR, {}).values()+ for adapter in adapters+ }+ return list(roots.values())+++def _refresh_group_offload(roots: list) -> None:+ """Re-apply group offloading on ``roots`` (as diffusers does after PEFT injection): groups list their tensors when+ applied, so LoKR buffers added or dropped since would be missed."""+ if not roots:+ return+ from diffusers.hooks.group_offloading import _maybe_remove_and_reapply_group_offloading++ for root in roots:+ _maybe_remove_and_reapply_group_offloading(root)+++def _unwrap_kron(pipe: Any) -> list:+ out = []+ seen: set = set()+ for adapters in pipe.__dict__.get(KRON_ATTR, {}).values():+ for adapter in adapters:+ wrapper = adapter.wrapper+ if id(wrapper) in seen:+ continue+ seen.add(id(wrapper))+ parent, attr = _parent_of(adapter.root, wrapper)+ setattr(parent, attr, wrapper.base_layer)+ out.append((parent, attr, wrapper))+ return out+++def _rewrap_kron(wrappers: list) -> None:+ for parent, attr, wrapper in wrappers:+ wrapper.base_layer = getattr(parent, attr)+ setattr(parent, attr, wrapper)+++def kron_names(pipe: Any) -> tuple[str, ...]:+ return tuple(pipe.__dict__.get(KRON_ATTR, {}))+++def set_adapter_weights(pipe: Any, names: list, weights: list) -> None:+ """``pipe.set_adapters`` for both kinds; a LoKR-only name has no PEFT adapter to set."""+ set_kron_weights(pipe, dict(zip(names, weights)))+ kron_only = pipe.__dict__.get(KRON_ONLY_ATTR, set())+ pairs = [(n, w) for n, w in zip(names, weights) if n not in kron_only]+ if pairs or not kron_only:+ pipe.set_adapters([n for n, _w in pairs], adapter_weights = [w for _n, w in pairs])+++def kron_only_selection(pipe: Any) -> bool:+ names = [entry[0] for entry in getattr(pipe, "_unsloth_loras", ()) or ()]+ kron_only = pipe.__dict__.get(KRON_ONLY_ATTR, set())+ return bool(names) and all(name in kron_only for name in names)+++def set_kron_weights(pipe: Any, weights: dict) -> None:+ """Every LoKR adapter's weight (name -> weight; an omitted name is 0, as a pinned PEFT scale is)."""+ for adapter_name, adapters in pipe.__dict__.get(KRON_ATTR, {}).items():+ for adapter in adapters:+ adapter.set_weight(float(weights.get(adapter_name, 0.0)))+++def unload_adapters(pipe: Any) -> None:+ roots = _kron_roots(pipe)+ for adapters in pipe.__dict__.pop(KRON_ATTR, {}).values():+ for adapter in adapters:+ adapter.remove()+ pipe.__dict__.pop(KRON_ONLY_ATTR, None)+ pipe.unload_lora_weights()+ _refresh_group_offload(roots)diff --git a/studio/backend/core/inference/diffusion_moss_image.py b/studio/backend/core/inference/diffusion_moss_image.pyindex 2a907729b14..df405e5470d 100644--- a/studio/backend/core/inference/diffusion_moss_image.py+++ b/studio/backend/core/inference/diffusion_moss_image.py@@ -334,6 +334,7 @@ def _model_bases():_ModelMixin, _ConfigMixin = _model_bases()+from diffusers.loaders import PeftAdapterMixin as _PeftAdapterMixin # noqa: E402 (LoRA: load_lora_adapter)from diffusers.configuration_utils import register_to_config # noqa: E402@@ -346,7 +347,7 @@ def _float64_device(device):return torch.device("cpu")-class MOSSImageTransformer(_ModelMixin, _ConfigMixin):+class MOSSImageTransformer(_ModelMixin, _ConfigMixin, _PeftAdapterMixin):"""MOSS-Image DiT: a text refiner, then 42 single-stream joint blocks over [image ; text] sharing one AdaLN.Each sample runs at its true text length, text refinement included, so padded tokens never reach attention."""@@ -688,6 +689,47 @@ class MOSSImagePipeline(DiffusionPipeline):def interrupt(self) -> bool:return self._interrupt+ # LoRA on the transformer only; diffusion_lora_files folds alpha and splits fused qkv first.+ def load_lora_weights(+ self,+ state_dict,+ adapter_name: Optional[str] = None,+ **_kwargs,+ ):+ metadata = None+ if not isinstance(state_dict, dict):+ from diffusers.utils.state_dict_utils import _load_sft_state_dict_metadata+ from safetensors.torch import load_file++ # lora_alpha / rank of a diffusers-saved adapter live only in the file's metadata.+ metadata = _load_sft_state_dict_metadata(str(state_dict))+ state_dict = load_file(str(state_dict))+ converted = {}+ for key, value in state_dict.items():+ for prefix in ("diffusion_model.", "model.diffusion_model.", "transformer."):+ if key.startswith(prefix):+ key = key[len(prefix) :]+ break+ converted["transformer." + key] = value+ self.transformer.load_lora_adapter(+ converted, adapter_name = adapter_name, prefix = "transformer", metadata = metadata+ )++ def set_adapters(+ self,+ adapter_names,+ adapter_weights = None,+ ):+ self.transformer.set_adapters(adapter_names, weights = adapter_weights)++ def get_list_adapters(self) -> dict:+ config = getattr(self.transformer, "peft_config", None) or {}+ return {"transformer": list(config)} if config else {}++ def unload_lora_weights(self):+ if getattr(self.transformer, "peft_config", None):+ self.transformer.unload_lora()+@torch.no_grad()def encode_prompt(self,diff --git a/studio/backend/tests/test_diffusion_lora.py b/studio/backend/tests/test_diffusion_lora.pyindex 66807d58160..c4c03edd0f8 100644--- a/studio/backend/tests/test_diffusion_lora.py+++ b/studio/backend/tests/test_diffusion_lora.py@@ -157,8 +157,11 @@ def test_lora_unavailable_reason_names_the_cause():assert "NVFP4" in dl.lora_unavailable_reason(engine = "diffusers", family = "flux.1", model_kind = "pipeline", transformer_quant = "nvfp4")- assert "no LoRA loader" in dl.lora_unavailable_reason(- engine = "diffusers", family = "moss-image", model_kind = "pipeline", transformer_quant = None+ assert (+ dl.lora_unavailable_reason(+ engine = "diffusers", family = "moss-image", model_kind = "pipeline", transformer_quant = None+ )+ is None)assert "Speed" in dl.lora_unavailable_reason(engine = "diffusers",@@ -507,6 +510,26 @@ def test_baked_quant_pipe_disables_any_subset_of_its_adapters(monkeypatch):assert pipe.active == (["style", "style_2"], [0.7, 1.0])+def test_a_declined_quantize_keeps_the_dense_lora_path(monkeypatch):+ import threading++ monkeypatch.setattr(+ dl,+ "resolve_specs",+ lambda specs, **_: [+ dl.ResolvedLora(i, dl.sanitize_alias(i), f"/{i}.safetensors", "safetensors", w)+ for i, w in specs+ ],+ )+ b = _backend()+ pipe = _FakePipe()+ pipe._unsloth_loras = b._resolve_lora_set([("styleA", 1.0)], family = "flux.1", hf_token = None)+ pipe._unsloth_loras_baked = True+ b._apply_loras(_fake_state(pipe), [("styleB", 1.0)], threading.Event())+ assert [n for _p, n in pipe.loaded] == ["styleB"]+ assert pipe.active == (["styleB"], [1.0])++@pytest.mark.parametrize("mode, refused", [("group", True), ("resident", False), ("model", False)])def test_baked_weights_stay_fixed_under_a_block_streamed_graph(monkeypatch, mode, refused):# Resetting a block-streamed graph for new LoRA scales left every later generate failing. Refuse the change.@@ -594,16 +617,29 @@ def test_scan_local_tolerates_bad_sidecar(tmp_path, monkeypatch):assert entry.families == () and entry.weight_default == 1.0-def test_moss_image_has_no_lora_loader_so_lora_is_gated_off(monkeypatch):+def test_moss_image_has_a_lora_loader():+ pytest.importorskip("torch")+ pytest.importorskip("diffusers")+ from core.inference.diffusion_moss_image import MOSSImagePipeline, MOSSImageTransformer++ for method in ("load_lora_weights", "set_adapters", "unload_lora_weights", "get_list_adapters"):+ assert hasattr(MOSSImagePipeline, method)+ assert hasattr(MOSSImageTransformer, "load_lora_adapter")+ assert dl.supports_lora(+ engine = "diffusers", family = "moss-image", model_kind = "pipeline", transformer_quant = None+ )+++def test_a_family_without_a_lora_loader_is_gated_off(monkeypatch):+ """The gate for an in-tree pipeline with no LoRA loader, exercised on MOSS-Image as if it still had none."""import threadingpytest.importorskip("torch")pytest.importorskip("diffusers")from core.inference.diffusion_families import detect_family- from core.inference.diffusion_moss_image import MOSSImagePipeline- assert not hasattr(MOSSImagePipeline, "load_lora_weights")+ monkeypatch.setattr(dl, "_DIFFUSERS_LORA_UNSUPPORTED_FAMILIES", ("moss-image",))for quant in (None, "int8", "fp8"):assert not dl.supports_lora(engine = "diffusers", family = "moss-image", model_kind = "pipeline", transformer_quant = quant@@ -1056,3 +1092,47 @@ def test_diffusers_weight_change_falls_back_to_reload_when_set_adapters_fails(assert pipe.unloaded == 1assert pipe.active == (["styleA"], [0.5])assert [(n, w) for n, _p, w in pipe._unsloth_loras] == [("styleA", 0.5)]++[email protected]("baked", [True, False])+def test_a_lokr_only_weight_change_keeps_the_cuda_graphs(monkeypatch, tmp_path, baked):+ """A LoKR-only strength change (or disable) writes scale tensors in place: graphs kept, CFG replica dropped."""+ import threading++ from core.inference import diffusion as diffusion_mod+ from core.inference import diffusion_lora_files as lf++ _real_file_specs(monkeypatch, tmp_path)+ resets: list = []+ invalidated: list = []+ monkeypatch.setattr(+ diffusion_mod.cuda_graph, "reset_all", lambda handles: resets.append(handles)+ )+ monkeypatch.setattr(+ diffusion_mod, "invalidate_cfg_replica", lambda pipe, why, log = None: invalidated.append(why)+ )+ # The scale mirror copies PEFT scales only, so even a mirror that lines up must not keep the replica.+ monkeypatch.setattr(+ diffusion_mod, "mirror_cfg_replica_adapter_scales", lambda pipe, log = None: True+ )+ torch = pytest.importorskip("torch")+ block = torch.nn.ModuleDict({"q": torch.nn.Linear(24, 36, bias = False)})+ adapter = lf.KronAdapter(block, block["q"], torch.randn(4, 3), torch.randn(9, 8), 1.0)+ pipe = _FakePipe()+ pipe.__dict__[lf.KRON_ATTR] = {"styleA": [adapter]}+ pipe.__dict__[lf.KRON_ONLY_ATTR] = {"styleA"}+ b = _backend()+ state = _fake_state(pipe, quant = "int8" if baked else None)+ pipe._unsloth_loras = b._resolve_lora_set([("styleA", 1.0)], family = "flux.1", hf_token = None)+ pipe._unsloth_loras_baked = baked+ if not baked:+ monkeypatch.setattr(diffusion_mod, "_lora_weights_only_change", lambda *a: True)+ monkeypatch.setattr(diffusion_mod, "_lora_load_block_reason", lambda pipe: None)++ b._apply_loras(state, [("styleA", 0.4)], threading.Event())+ assert float(getattr(block["q"], adapter.prefix + "_scale")) == pytest.approx(0.4)+ if baked:+ b._apply_loras(state, [], threading.Event()) # disabling every adapter+ assert float(getattr(block["q"], adapter.prefix + "_scale")) == 0.0+ assert resets == []+ assert invalidateddiff --git a/studio/backend/tests/test_diffusion_lora_files.py b/studio/backend/tests/test_diffusion_lora_files.pynew file mode 100644index 00000000000..11824dbbc7c--- /dev/null+++ b/studio/backend/tests/test_diffusion_lora_files.py@@ -0,0 +1,442 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""LoRA and LyCORIS LoKR files: key normalization and the Kronecker adapter."""++from __future__ import annotations++import pytest++torch = pytest.importorskip("torch")++from core.inference import diffusion_lora_files as lf # noqa: E402+++def test_alpha_folds_into_lora_b_and_down_up_rename():+ a, b = torch.randn(4, 8), torch.randn(6, 4)+ out = lf.normalize_lora(+ {+ "diffusion_model.blocks.0.attn.wq.lora_down.weight": a,+ "diffusion_model.blocks.0.attn.wq.lora_up.weight": b,+ "diffusion_model.blocks.0.attn.wq.alpha": torch.tensor(2.0),+ "diffusion_model.blocks.0.attn.wk.alpha": torch.tensor(+ 1.0+ ), # an orphan alpha is dropped+ },+ "krea-2",+ )+ assert sorted(out) == [+ "diffusion_model.blocks.0.attn.wq.lora_A.weight",+ "diffusion_model.blocks.0.attn.wq.lora_B.weight",+ ]+ assert torch.allclose(out["diffusion_model.blocks.0.attn.wq.lora_B.weight"], b * 0.5)+++def test_anima_embedding_pair_becomes_a_peft_embedding_lora():+ a, b = (+ torch.randn(3, 5),+ torch.randn(11, 3),+ ) # ComfyUI: delta = B @ A on the [vocab 11, dim 5] table+ out = lf.normalize_lora(+ {+ "diffusion_model.llm_adapter.embed.lora_A.weight": a,+ "diffusion_model.llm_adapter.embed.lora_B.weight": b,+ },+ "anima",+ )+ ea = out["diffusion_model.llm_adapter.embed.lora_embedding_A"]+ eb = out["diffusion_model.llm_adapter.embed.lora_embedding_B"]+ # PEFT's embedding delta is (B' @ A').T+ assert torch.allclose((eb @ ea).T, b @ a)+++def test_moss_fused_qkv_lora_splits_rows_and_shares_a():+ a, b = torch.randn(4, 6), torch.randn(9, 4)+ out = lf.normalize_lora(+ {+ "diffusion_model.layers.0.proj.qkv.lora_A.weight": a,+ "diffusion_model.layers.0.proj.qkv.lora_B.weight": b,+ },+ "moss-image",+ )+ for i, name in enumerate(("q", "k", "v")):+ assert torch.equal(out[f"diffusion_model.layers.0.proj.{name}.lora_A.weight"], a)+ assert torch.equal(+ out[f"diffusion_model.layers.0.proj.{name}.lora_B.weight"], b[i * 3 : (i + 1) * 3]+ )+++def test_split_lokr_keeps_alpha_with_its_module():+ w1, w2 = torch.randn(2, 2), torch.randn(3, 3)+ lokr, rest = lf.split_lokr(+ {+ "m.lokr_w1": w1,+ "m.lokr_w2": w2,+ "m.alpha": torch.tensor(4.0),+ "n.lora_A.weight": torch.randn(1, 6),+ "n.alpha": torch.tensor(1.0),+ }+ )+ assert set(lokr["m"]) == {"lokr_w1", "lokr_w2", "alpha"}+ assert set(rest) == {"n.lora_A.weight", "n.alpha"}+++def test_kron_factors_scale_like_comfyui():+ w1, w2 = torch.randn(2, 2), torch.randn(3, 3)+ # Full factors: alpha is ignored (scale 1).+ assert lf._kron_factors({"lokr_w1": w1, "lokr_w2": w2, "alpha": torch.tensor(8.0)})[2] == 1.0+ # A low-rank factor: alpha / its rank.+ _, _, scale = lf._kron_factors(+ {+ "lokr_w1": w1,+ "lokr_w2_a": torch.randn(3, 2),+ "lokr_w2_b": torch.randn(2, 3),+ "alpha": torch.tensor(1.0),+ }+ )+ assert scale == 0.5+++class _Block(torch.nn.Module):+ def __init__(self, out_features = 36):+ super().__init__()+ self.proj = torch.nn.ModuleDict({"q": torch.nn.Linear(24, out_features, bias = False)})++ def forward(self, x):+ return self.proj["q"](x)+++def test_kron_adapter_matches_the_dense_delta_and_restores_the_layer():+ torch.manual_seed(0)+ block = _Block().double()+ base = block.proj["q"]+ w1, w2 = torch.randn(4, 3).double(), torch.randn(9, 8).double()+ first = lf.KronAdapter(block, base, w1, w2, 1.0)+ second = lf.KronAdapter(block, block.proj["q"], w1, w2, 1.0)+ first.set_weight(0.7)+ second.set_weight(0.3)+ x = torch.randn(2, 5, 24, dtype = torch.float64)+ expected = x @ (base.weight + torch.kron(w1, w2)).T+ assert torch.allclose(block(x), expected, atol = 1e-10)+ assert block.proj["q"].weight is base.weight+ first.remove()+ second.remove()+ assert block.proj["q"] is base+++def test_kron_adapter_row_slice_for_a_fused_qkv_factor():+ torch.manual_seed(1)+ block = _Block(out_features = 12).double()+ w1, w2 = (+ torch.randn(4, 3).double(),+ torch.randn(9, 8).double(),+ ) # 36 rows, this layer takes rows 12..24+ lf.KronAdapter(block, block.proj["q"], w1, w2, 1.0, rows = (12, 24))+ x = torch.randn(3, 24, dtype = torch.float64)+ base = block.proj["q"].base_layer+ assert torch.allclose(block(x), x @ (base.weight + torch.kron(w1, w2)[12:24]).T, atol = 1e-10)+++def test_a_failed_lokr_allocation_leaves_the_layer_alone():+ class _OutOfMemory:+ def to(self, **_kwargs):+ raise RuntimeError("CUDA out of memory")++ block = _Block()+ base = block.proj["q"]+ with pytest.raises(RuntimeError, match = "out of memory"):+ lf.KronAdapter(block, base, torch.randn(4, 3), _OutOfMemory(), 1.0)+ assert block.proj["q"] is base+++def test_the_moss_loader_keeps_the_adapter_metadata(tmp_path):+ pytest.importorskip("diffusers")+ import json+ from types import SimpleNamespace++ from safetensors.torch import save_file++ from core.inference.diffusion_moss_image import MOSSImagePipeline++ path = tmp_path / "moss.safetensors"+ meta = {"transformer.r": 2, "transformer.lora_alpha": 8}+ save_file(+ {+ "transformer.layers.0.proj.q.lora_A.weight": torch.randn(2, 4),+ "transformer.layers.0.proj.q.lora_B.weight": torch.randn(4, 2),+ },+ str(path),+ metadata = {"lora_adapter_metadata": json.dumps(meta)},+ )+ calls = []+ pipe = SimpleNamespace(+ transformer = SimpleNamespace(load_lora_adapter = lambda sd, **kw: calls.append((sd, kw)))+ )+ MOSSImagePipeline.load_lora_weights(pipe, str(path), adapter_name = "m")+ ((state, kwargs),) = calls+ assert sorted(state) == [+ "transformer.layers.0.proj.q.lora_A.weight",+ "transformer.layers.0.proj.q.lora_B.weight",+ ]+ assert kwargs["metadata"] == meta+++def test_peft_loads_around_an_existing_lokr_wrapper():+ """A LoRA added after a LoKR on one layer: PEFT injects into the bare Linear, then the wrapper goes back on."""+ pytest.importorskip("peft")+ from core.inference.diffusion_torchao_patches import install_peft_torchao_dispatch_guard++ install_peft_torchao_dispatch_guard() # as every diffusion entry point does+ block = _Block().double()++ class _Pipe:+ def __init__(self, transformer):+ self.transformer = transformer++ def load_lora_weights(+ self,+ _state,+ adapter_name = None,+ ):+ from peft import LoraConfig, inject_adapter_in_model+ inject_adapter_in_model(+ LoraConfig(r = 2, target_modules = ["q"]), self.transformer, adapter_name+ )++ pipe = _Pipe(block)+ adapter = lf.KronAdapter(+ block, block.proj["q"], torch.randn(4, 3).double(), torch.randn(9, 8).double(), 1.0+ )+ pipe.__dict__[lf.KRON_ATTR] = {"k": [adapter]}+ wrappers = lf._unwrap_kron(pipe)+ pipe.load_lora_weights({}, adapter_name = "l")+ lf._rewrap_kron(wrappers)+ wrapper = block.proj["q"]+ assert isinstance(wrapper, lf.kron_linear_class())+ assert type(wrapper.base_layer).__name__ == "Linear" and hasattr(wrapper.base_layer, "lora_A")+++def test_an_embedding_lokr_is_refused_at_load(tmp_path):+ """A LoKR on Anima's ``llm_adapter.embed`` table (token ids, not features) fails at load, not at render."""+ conversion = pytest.importorskip("diffusers.loaders.lora_conversion_utils")+ if not hasattr(conversion, "_convert_non_diffusers_anima_lora_to_diffusers"):+ pytest.skip("this diffusers has no Anima LoRA converter")+ from safetensors.torch import save_file++ class _Conditioner(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.embed = torch.nn.Embedding(12, 6)++ class _Pipe:+ def __init__(self):+ self.text_conditioner = _Conditioner()++ pipe = _Pipe()+ embed = pipe.text_conditioner.embed+ path = tmp_path / "lokr.safetensors"+ # A [12, 6] table: w1 [3, 2], w2 [4, 3].+ save_file(+ {+ "diffusion_model.llm_adapter.embed.lokr_w1": torch.randn(3, 2),+ "diffusion_model.llm_adapter.embed.lokr_w2": torch.randn(4, 3),+ },+ path,+ )+ with pytest.raises(ValueError, match = "embedding"):+ lf.load_adapter(pipe, str(path), "k", "anima")+ assert pipe.text_conditioner.embed is embed+ assert lf.kron_names(pipe) == ()+++class _Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.blocks = torch.nn.ModuleList([_Block(), _Block(24)])+++class _RecordingPipe:+ def __init__(self, loads = True):+ self.transformer = _Net()+ self.loads = loads+ self.loaded: dict = {}++ def load_lora_weights(+ self,+ state,+ adapter_name = None,+ ):+ if self.loads:+ self.loaded[adapter_name] = dict(state)++ def get_list_adapters(self):+ return {"transformer": list(self.loaded)} if self.loaded else {}+++def test_kohya_keys_resolve_to_the_layer_they_name(tmp_path, monkeypatch):+ from safetensors.torch import save_file++ monkeypatch.setattr(lf, "_refresh_group_offload", lambda roots: None) # no diffusers needed++ a, b = torch.randn(2, 24), torch.randn(24, 2)+ path = tmp_path / "kohya.safetensors"+ save_file(+ {+ "lora_unet_blocks_1_proj_q.lora_down.weight": a,+ "lora_unet_blocks_1_proj_q.lora_up.weight": b,+ "lora_unet_blocks_1_proj_q.alpha": torch.tensor(1.0),+ "lycoris_blocks_0_proj_q.lokr_w1": torch.randn(4, 3),+ "lycoris_blocks_0_proj_q.lokr_w2": torch.randn(9, 8),+ },+ path,+ )+ pipe = _RecordingPipe()+ lf.load_adapter(pipe, str(path), "k", "moss-image")+ loaded = pipe.loaded["k"]+ assert sorted(loaded) == [+ "diffusion_model.blocks.1.proj.q.lora_A.weight",+ "diffusion_model.blocks.1.proj.q.lora_B.weight",+ ]+ assert torch.allclose(loaded["diffusion_model.blocks.1.proj.q.lora_B.weight"], b * 0.5)+ assert isinstance(pipe.transformer.blocks[0].proj["q"], lf.kron_linear_class())+++def test_a_file_needing_no_rewrite_loads_by_path(tmp_path):+ from safetensors.torch import save_file++ diffusers_format = tmp_path / "diffusers.safetensors"+ save_file(+ {+ "transformer.blocks.1.proj.q.lora_A.weight": torch.randn(2, 24),+ "transformer.blocks.1.proj.q.lora_B.weight": torch.randn(24, 2),+ },+ diffusers_format,+ metadata = {"lora_adapter_metadata": '{"r": 2, "lora_alpha": 8}'},+ )+ pipe = _RecordingPipe()+ received = []+ pipe.load_lora_weights = lambda source, adapter_name = None: (+ received.append(source),+ pipe.loaded.setdefault(adapter_name, {}),+ )+ lf.load_adapter(pipe, str(diffusers_format), "d", "krea-2")+ assert received == [str(diffusers_format)]+ reference_format = tmp_path / "reference.safetensors"+ save_file(+ {+ "diffusion_model.blocks.1.proj.q.lora_down.weight": torch.randn(2, 24),+ "diffusion_model.blocks.1.proj.q.lora_up.weight": torch.randn(24, 2),+ },+ reference_format,+ )+ lf.load_adapter(pipe, str(reference_format), "r", "krea-2")+ assert isinstance(received[-1], dict)+++def test_a_lora_that_loads_into_no_layer_is_refused(tmp_path):+ from safetensors.torch import save_file++ path = tmp_path / "te_only.safetensors"+ save_file(+ {+ "lora_te_layers_0_mlp_up.lora_down.weight": torch.randn(2, 8),+ "lora_te_layers_0_mlp_up.lora_up.weight": torch.randn(8, 2),+ },+ path,+ )+ with pytest.raises(ValueError, match = "none of its LoRA keys"):+ lf.load_adapter(_RecordingPipe(loads = False), str(path), "k", "moss-image")+++def test_a_dora_lokr_is_refused(tmp_path):+ from safetensors.torch import save_file++ path = tmp_path / "dora.safetensors"+ save_file(+ {+ "diffusion_model.blocks.0.proj.q.lokr_w1": torch.randn(4, 3),+ "diffusion_model.blocks.0.proj.q.lokr_w2": torch.randn(9, 8),+ "diffusion_model.blocks.0.proj.q.dora_scale": torch.ones(1, 24),+ },+ path,+ )+ pipe = _RecordingPipe()+ with pytest.raises(ValueError, match = "DoRA"):+ lf.load_adapter(pipe, str(path), "k", "moss-image")+ assert lf.kron_names(pipe) == ()+++def test_compiled_block_reads_a_new_lokr_weight():+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA")+ block = _Block().cuda().bfloat16()+ adapter = lf.KronAdapter(block, block.proj["q"], torch.randn(4, 3), torch.randn(9, 8), 1.0)+ block.compile(fullgraph = True)+ x = torch.randn(2, 24, device = "cuda", dtype = torch.bfloat16)+ on = block(x)+ adapter.set_weight(0.0)+ off = block(x)+ assert (on - off).abs().max() > 0+ assert torch.allclose(off, x @ block.proj["q"].weight.T)+++def test_moss_and_anima_have_a_lora_loader():+ from core.inference.diffusion_lora import diffusers_lora_family_supported+ assert diffusers_lora_family_supported("moss-image")+ assert diffusers_lora_family_supported("anima")++[email protected]("use_stream", [False, True])+def test_lokr_attached_under_group_offload_moves_with_its_group(tmp_path, use_stream):+ """A LoKR attached at generate time to a group-offloaded transformer: the groups are rebuilt, so the factors onload+ with their block, and again on unload."""+ if not torch.cuda.is_available():+ pytest.skip("needs CUDA")+ from diffusers.hooks import apply_group_offloading+ from safetensors.torch import save_file++ torch.manual_seed(0)++ class _Net(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.blocks = torch.nn.ModuleList([_Block(), _Block(24)])++ def forward(self, x):+ return self.blocks[0](self.blocks[1](x))++ class _Pipe:+ def __init__(self, transformer):+ self.transformer = transformer++ def unload_lora_weights(self):+ pass++ net = _Net().bfloat16()+ apply_group_offloading(+ net,+ onload_device = torch.device("cuda"),+ offload_device = torch.device("cpu"),+ offload_type = "block_level",+ num_blocks_per_group = 1,+ use_stream = use_stream,+ )+ x = torch.randn(2, 24, device = "cuda", dtype = torch.bfloat16)+ bare = net(x)+ w1, w2 = torch.randn(4, 3), torch.randn(9, 8)+ path = tmp_path / "lokr.safetensors"+ save_file(+ {+ "diffusion_model.blocks.0.proj.q.lokr_w1": w1,+ "diffusion_model.blocks.0.proj.q.lokr_w2": w2,+ },+ path,+ )+ pipe = _Pipe(net)+ lf.load_adapter(pipe, str(path), "k", "moss-image")+ out = net(x)+ inner = net.blocks[1](x)+ expected = bare + (inner @ torch.kron(w1, w2).to("cuda", torch.bfloat16).T)+ assert torch.allclose(out.float(), expected.float(), atol = 0.25, rtol = 0.05)+ lf.unload_adapters(pipe)+ assert torch.equal(net(x), bare)