Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: apply ComfyUI LoRA and LoKR files to Anima, Krea 2 and MOSS-Image (#16711)

oobabooga083ba2f8c122
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index a045e158cfd..7ff5c699ab5 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -393,6 +393,7 @@ from .diffusion_precision import (
torchao_quantize_importable,
)
from .diffusion_te_prequant import supplied_component_pipe_kwargs, te_prequant_pipe_kwargs
+from . import diffusion_lora_files as lora_files
from .diffusion_fast_load import _snapshot_dir as _load_snapshot_dir
from .diffusion_fast_load import start_load_prefetch, stop_prefetch, te_precast_components
from .diffusion_hosted_vae import hosted_vae_pipe_kwargs, hosted_vae_plan_files
@@ -897,6 +898,8 @@ def _baked_lora_names(pipe: Any) -> list:
return names
+# Adapters baked into a quantized single file; ``_unsloth_loras_baked`` also stays set after a declined quantize.
+_QUANT_FILE_LORA_BAKE_ATTR = "_unsloth_quant_file_loras_baked"
# ``(module, adapter, base)`` for every baked LoRA scale held as a device tensor; ``base`` is lora_alpha / r.
_PINNED_LORA_SCALES_ATTR = "_unsloth_pinned_lora_scales"
# ``(module, adapter)`` for the baked LoRA layers OUTSIDE the denoisers (a text encoder carrying the same file's
@@ -994,9 +997,16 @@ def _set_pinned_lora_scales(pipe: Any, weights: dict) -> bool:
weights): the caller then takes the set_adapters + graph reset path."""
pinned = getattr(pipe, _PINNED_LORA_SCALES_ATTR, None)
if not pinned:
+ if lora_files.kron_only_selection(pipe):
+ # LoKR scales are tensors written in place; the CFG replica holds copies, so it is rebuilt.
+ lora_files.set_kron_weights(pipe, weights)
+ invalidate_cfg_replica(pipe, "the LoKR weights changed")
+ return True
return False
import torch
+ lora_files.set_kron_weights(pipe, weights)
+
for module, adapter, _base in pinned:
value = module.scaling.get(adapter)
if (
@@ -10267,10 +10277,13 @@ class DiffusionBackend:
hf_token = hf_token,
)
for name, path, _weight in baked:
- pipe.load_lora_weights(path, adapter_name = name)
- pipe.set_adapters(
+ lora_files.load_adapter(
+ pipe, path, name, getattr(fam, "name", None)
+ )
+ lora_files.set_adapter_weights(
+ pipe,
[n for (n, _p, _w) in baked],
- adapter_weights = [w for (_n, _p, w) in baked],
+ [w for (_n, _p, w) in baked],
)
pipe._unsloth_loras = baked
pipe._unsloth_loras_baked = True
@@ -10409,6 +10422,28 @@ class DiffusionBackend:
effective_speed = resolve_speed_mode(speed_mode, is_gguf = kind == "gguf")
split_expected = _layer_split_expected(pipe, plan)
# A torchao (or torchao-free resident) int8 DiT must be compiled: eager is ~30x slower, losing to GGUF.
+ if comfy_compile and _has_active_lora(loras):
+ # A quantized single file bakes its adapters before the compile; a new adapter set reloads.
+ baked = self._resolve_lora_set(
+ [(i, w) for (i, w) in loras if w != 0],
+ family = getattr(fam, "name", None),
+ hf_token = hf_token,
+ )
+ for name, path, _weight in baked:
+ lora_files.load_adapter(pipe, path, name, getattr(fam, "name", None))
+ lora_files.set_adapter_weights(
+ pipe,
+ [n for (n, _p, _w) in baked],
+ [w for (_n, _p, w) in baked],
+ )
+ pipe._unsloth_loras = baked
+ pipe._unsloth_loras_baked = True
+ setattr(pipe, _QUANT_FILE_LORA_BAKE_ATTR, True)
+ logger.info(
+ "diffusion.lora_bake: %d adapter(s) attached to the ComfyUI-quantized transformer "
+ "before the compile",
+ len(baked),
+ )
quant_forces_compile = (
transformer_quant_engaged is not None or comfy_compile
) and (
@@ -11684,11 +11719,12 @@ class DiffusionBackend:
)
for name, path, _weight in baked:
check_cancelled()
- pipe.load_lora_weights(path, adapter_name = name)
+ lora_files.load_adapter(pipe, path, name, getattr(fam, "name", None))
check_cancelled()
- pipe.set_adapters(
+ lora_files.set_adapter_weights(
+ pipe,
[n for (n, _p, _w) in baked],
- adapter_weights = [w for (_n, _p, w) in baked],
+ [w for (_n, _p, w) in baked],
)
pipe._unsloth_loras = baked
pipe._unsloth_loras_baked = True
@@ -13522,7 +13558,10 @@ class DiffusionBackend:
_refuse_lora_without_loader(getattr(state, "family", None), specs)
quant_baked = bool(getattr(pipe, "_unsloth_loras_baked", False))
quant = (state.transformer_quant or "").lower()
- if quant in ("int8", "fp8", "nvfp4", "mxfp8"):
+ # A quantized single file reports no transformer_quant but bakes its adapters the same way.
+ if quant in ("int8", "fp8", "nvfp4", "mxfp8") or getattr(
+ pipe, _QUANT_FILE_LORA_BAKE_ATTR, False
+ ):
self._adjust_baked_loras(state, pipe, specs, current, quant_baked, cancel)
return
@@ -13530,7 +13569,7 @@ class DiffusionBackend:
if current:
self._unfuse_loras(state, pipe)
try:
- pipe.unload_lora_weights()
+ lora_files.unload_adapters(pipe)
except Exception: # noqa: BLE001 -- best-effort clear
pass
pipe._unsloth_loras = ()
@@ -13568,15 +13607,17 @@ class DiffusionBackend:
weights = [w for _n, _p, w in desired]
if _lora_weights_only_change(state, pipe, current, desired):
try:
- pipe.set_adapters(names, adapter_weights = weights)
+ lora_files.set_adapter_weights(pipe, names, weights)
except Exception: # noqa: BLE001 -- the full reload below sets the same scales or fails cleanly
pass
else:
pipe._unsloth_loras = desired
- # A captured graph bakes the scales in.
- cuda_graph.reset_all(getattr(state, "cuda_graphs", ()))
- # Same adapters, new scales: the replica takes the scales; a rebuild only when they do not line up.
- if not mirror_cfg_replica_adapter_scales(pipe, logger):
+ # A captured graph bakes PEFT's float scales in; LoKR scales are tensors it reads at run time.
+ if not lora_files.kron_only_selection(pipe):
+ cuda_graph.reset_all(getattr(state, "cuda_graphs", ()))
+ # The scale mirror skips LoKR buffers: LoKR or misaligned layers rebuild the replica.
+ kron = lora_files.kron_names(pipe)
+ if kron or not mirror_cfg_replica_adapter_scales(pipe, logger):
invalidate_cfg_replica(pipe, "the LoRA adapters changed", logger)
return
# PEFT targets to_q / to_k / to_v: a fused U-Net that dropped them gets them back first (bit-exact).
@@ -13585,14 +13626,14 @@ class DiffusionBackend:
file_sigs = _lora_file_sigs(desired)
try:
if current:
- pipe.unload_lora_weights()
+ lora_files.unload_adapters(pipe)
setattr(pipe, _LORA_FILE_SIGS_ATTR, None)
for name, path, _weight in desired:
- pipe.load_lora_weights(path, adapter_name = name)
- pipe.set_adapters(names, adapter_weights = weights)
+ lora_files.load_adapter(pipe, path, name, getattr(state.family, "name", None))
+ lora_files.set_adapter_weights(pipe, names, weights)
except Exception as exc: # noqa: BLE001 -- surface as a clean 400
try:
- pipe.unload_lora_weights()
+ lora_files.unload_adapters(pipe)
except Exception: # noqa: BLE001
pass
pipe._unsloth_loras = ()
@@ -13664,7 +13705,7 @@ class DiffusionBackend:
if _set_pinned_lora_scales(pipe, {}):
pipe._unsloth_loras = disabled
return
- pipe.set_adapters(names, adapter_weights = [0.0] * len(names))
+ lora_files.set_adapter_weights(pipe, names, [0.0] * len(names))
pipe._unsloth_loras = disabled
# Adapter scales are Python values read inside the LoRA forward, so a graph bakes them in.
cuda_graph.reset_all(getattr(state, "cuda_graphs", ()))
@@ -13696,9 +13737,10 @@ class DiffusionBackend:
if _set_pinned_lora_scales(pipe, {n: w for (n, _p, w) in updated}):
pipe._unsloth_loras = updated
return
- pipe.set_adapters(
+ lora_files.set_adapter_weights(
+ pipe,
[n for (n, _p, _w) in updated],
- adapter_weights = [w for (_n, _p, w) in updated],
+ [w for (_n, _p, w) in updated],
)
pipe._unsloth_loras = updated
cuda_graph.reset_all(getattr(state, "cuda_graphs", ()))
@@ -15626,7 +15668,10 @@ class DiffusionBackend:
model_kind = state.kind,
transformer_quant = state.transformer_quant,
)
- compiled = "compiled" in (getattr(state, "speed_optims", ()) or ())
+ # Baked adapters ride inside the compiled build.
+ compiled = "compiled" in (getattr(state, "speed_optims", ()) or ()) and not getattr(
+ state.pipe, _QUANT_FILE_LORA_BAKE_ATTR, False
+ )
lora_block = _lora_load_block_reason(state.pipe)
lora_load_ok = lora_block is None
lora_reason = lora_block or diffusion_lora.lora_unavailable_reason(
diff --git a/studio/backend/core/inference/diffusion_lora.py b/studio/backend/core/inference/diffusion_lora.py
index 57a0b35cb65..a036e4b6651 100644
--- a/studio/backend/core/inference/diffusion_lora.py
+++ b/studio/backend/core/inference/diffusion_lora.py
@@ -670,7 +670,7 @@ _DIFFUSERS_LORA_BAKED_QUANT = ("int8", "fp8")
# Prototype schemes with no validated LoRA path (and no shipped families needing one).
_DIFFUSERS_LORA_BLOCKED_QUANT = ("nvfp4", "mxfp8")
# In-tree pipelines with no LoRA loader mixin (no load_lora_weights / set_adapters / unload_lora_weights).
-_DIFFUSERS_LORA_UNSUPPORTED_FAMILIES = ("moss-image",)
+_DIFFUSERS_LORA_UNSUPPORTED_FAMILIES: tuple[str, ...] = ()
def diffusers_lora_family_supported(family: Optional[str]) -> bool:
diff --git a/studio/backend/core/inference/diffusion_lora_files.py b/studio/backend/core/inference/diffusion_lora_files.py
new file mode 100644
index 00000000000..30319586442
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_lora_files.py
@@ -0,0 +1,510 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""LoRA and LyCORIS LoKR files on Anima, Krea 2 and MOSS-Image. LoKR runs as ``KronLinear`` slots sharing PEFT's
+adapter names: :func:`set_adapter_weights` / :func:`unload_adapters` replace diffusers' adapter calls."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Optional
+
+from loggers import get_logger
+
+logger = get_logger(__name__)
+
+KRON_ATTR = "_unsloth_kron_adapters"
+# Adapter names whose file held LoKR modules only: diffusers has no PEFT adapter of that name to set.
+KRON_ONLY_ATTR = "_unsloth_kron_only"
+_PREFIXES = ("diffusion_model.", "model.diffusion_model.", "transformer.")
+# Kohya / SimpleTuner keys: the reference layer path with its dots written as underscores.
+_KOHYA_PREFIXES = ("lora_unet_", "lycoris_")
+_LOKR_SUFFIXES = (
+ "lokr_w1",
+ "lokr_w2",
+ "lokr_w1_a",
+ "lokr_w1_b",
+ "lokr_w2_a",
+ "lokr_w2_b",
+ "lokr_t2",
+)
+NORMALIZED_FAMILIES = ("anima", "krea-2", "moss-image")
+
+
+def read_lora_file(path: str) -> dict:
+ from safetensors.torch import load_file
+ return load_file(str(path))
+
+
+def _strip(key: str) -> str:
+ for prefix in _PREFIXES:
+ if key.startswith(prefix):
+ return key[len(prefix) :]
+ return key
+
+
+def _lokr_stem(key: str) -> Optional[str]:
+ for suffix in _LOKR_SUFFIXES:
+ if key.endswith("." + suffix):
+ return key[: -len(suffix) - 1]
+ return None
+
+
+def split_lokr(state: dict) -> tuple[dict, dict]:
+ """``(lokr, rest)``: LoKR tensors by module stem (each with its ``.alpha``), and every other tensor."""
+ lokr: dict[str, dict] = {}
+ for key, value in state.items():
+ stem = _lokr_stem(key)
+ if stem is not None:
+ lokr.setdefault(stem, {})[key[len(stem) + 1 :]] = value
+ rest = {}
+ for key, value in state.items():
+ if _lokr_stem(key) is not None:
+ continue
+ stem, _, suffix = key.rpartition(".")
+ if suffix in ("alpha", "dora_scale") and stem in lokr:
+ lokr[stem][suffix] = value
+ continue
+ rest[key] = value
+ return lokr, rest
+
+
+def normalize_lora(state: dict, family: Optional[str]) -> dict:
+ """Plain LoRA tensors as ``lora_A`` / ``lora_B`` with ``.alpha`` folded into ``lora_B`` (alpha / rank)."""
+ import torch
+
+ renamed: dict = {}
+ alphas: dict = {}
+ for key, value in state.items():
+ if key.endswith(".alpha"):
+ alphas[key[: -len(".alpha")]] = value
+ continue
+ key = key.replace(".lora_down.", ".lora_A.").replace(".lora_up.", ".lora_B.")
+ renamed[key] = value
+ for stem, alpha in alphas.items():
+ a, b = renamed.get(stem + ".lora_A.weight"), renamed.get(stem + ".lora_B.weight")
+ if a is None or b is None:
+ continue # an alpha with no low-rank pair (some converters write it for every module)
+ scale = float(alpha) / int(a.shape[0])
+ if scale != 1.0:
+ renamed[stem + ".lora_B.weight"] = (b.to(torch.float32) * scale).to(b.dtype)
+ if family == "anima":
+ renamed = _anima_embedding(renamed)
+ elif family == "moss-image":
+ renamed = _moss_fused_qkv(renamed)
+ return renamed
+
+
+def _anima_embedding(state: dict) -> dict:
+ """An ``llm_adapter.embed`` pair is a Linear delta ``B @ A`` on the ``[vocab, dim]`` embedding
+ table; PEFT's embedding LoRA computes ``(B' @ A').T``, so ``A' = B.T`` and ``B' = A.T``."""
+ out = dict(state)
+ for key in [k for k in state if ".llm_adapter.embed.lora_A.weight" in k]:
+ stem = key[: -len(".lora_A.weight")]
+ a, b = out.pop(stem + ".lora_A.weight"), out.pop(stem + ".lora_B.weight", None)
+ if b is None:
+ continue
+ out[stem + ".lora_embedding_A"] = b.T.contiguous()
+ out[stem + ".lora_embedding_B"] = a.T.contiguous()
+ return out
+
+
+_MOSS_QKV = re.compile(r"^(.*\.proj\.)qkv\.(lora_[AB])\.weight$")
+
+
+def _moss_fused_qkv(state: dict) -> dict:
+ """The fused ``proj.qkv`` as the in-tree ``proj.q`` / ``k`` / ``v``: ``A`` is shared, ``B``'s rows split."""
+ out: dict = {}
+ for key, value in state.items():
+ match = _MOSS_QKV.match(key)
+ if match is None:
+ out[key] = value
+ continue
+ stem, part = match.groups()
+ if part == "lora_A":
+ for name in ("q", "k", "v"):
+ out[f"{stem}{name}.lora_A.weight"] = value
+ else:
+ rows = int(value.shape[0]) // 3
+ for i, name in enumerate(("q", "k", "v")):
+ out[f"{stem}{name}.lora_B.weight"] = value[i * rows : (i + 1) * rows].contiguous()
+ return out
+
+
+def _kron_delta(x: Any, w1: Any, w2: Any, rows: Optional[tuple[int, int]]) -> Any:
+ """``x @ kron(w1, w2)[rows].T`` without the dense product: row ``i * b1 + k`` and column ``j * b2 + l`` of the
+ Kronecker product is ``w1[i, j] * w2[k, l]``, so two small matmuls on a reshaped input give it."""
+ a1, a2 = w1.shape
+ b1, b2 = w2.shape
+ lead = x.shape[:-1]
+ t = x.reshape(*lead, a2, b2) @ w2.T # (..., a2, b1)
+ y = (w1 @ t).reshape(*lead, a1 * b1) # (..., a1, b1) -> rows i * b1 + k
+ return y if rows is None else y[..., rows[0] : rows[1]]
+
+
+def _module_class():
+ from torch import nn
+ class KronLinear(nn.Module):
+ """A Linear plus LoKR deltas, a module rather than a hook (fused int8 paths and compiled forwards skip hooks).
+ Each 0-d ``scale`` buffer is filled in place, so compiled code and CUDA graphs need no retrace."""
+
+ def __init__(self, base_layer):
+ super().__init__()
+ self.base_layer = base_layer
+ self.slots: list = []
+
+ @property
+ def weight(self):
+ return self.base_layer.weight
+
+ @property
+ def bias(self):
+ return self.base_layer.bias
+
+ def get_base_layer(self):
+ base = self.base_layer
+ get = getattr(base, "get_base_layer", None)
+ return get() if callable(get) else base
+
+ def forward(self, x, *args, **kwargs):
+ out = self.base_layer(x, *args, **kwargs)
+ for prefix, rows in self.slots:
+ w1 = getattr(self, prefix + "_w1").to(x.dtype)
+ w2 = getattr(self, prefix + "_w2").to(x.dtype)
+ out = out + _kron_delta(x, w1, w2, rows) * getattr(self, prefix + "_scale").to(
+ out.dtype
+ )
+ return out
+
+ return KronLinear
+
+
+_KRON_CLASS = None
+
+
+def kron_linear_class():
+ global _KRON_CLASS
+ if _KRON_CLASS is None:
+ _KRON_CLASS = _module_class()
+ return _KRON_CLASS
+
+
+def _parent_of(root: Any, module: Any) -> tuple[Any, str]:
+ for name, child in root.named_modules():
+ for attr, sub in child.named_children():
+ if sub is module:
+ return child, attr
+ raise ValueError("the LoKR target is not a submodule of its component")
+
+
+class KronAdapter:
+ def __init__(
+ self,
+ root: Any,
+ module: Any,
+ w1: Any,
+ w2: Any,
+ alpha_scale: float,
+ rows: Optional[tuple[int, int]] = None,
+ ):
+ import torch
+
+ cls = kron_linear_class()
+ base = _base_layer(module)
+ device = _device_of(base)
+ dtype = _dtype_of(base)
+ self.alpha_scale = float(alpha_scale)
+ # Allocated before the layer is swapped, so a failure (OOM) leaves the model as it was.
+ buffers = (
+ w1.to(device = device, dtype = dtype),
+ w2.to(device = device, dtype = dtype),
+ torch.tensor(self.alpha_scale, dtype = torch.float32, device = device),
+ )
+ if not isinstance(module, cls):
+ parent, attr = _parent_of(root, module)
+ wrapper = cls(module)
+ setattr(parent, attr, wrapper)
+ module = wrapper
+ self.wrapper = module
+ self.root = root
+ self.prefix = f"kron{len(module.slots)}"
+ for suffix, tensor in zip(("_w1", "_w2", "_scale"), buffers):
+ module.register_buffer(self.prefix + suffix, tensor, persistent = False)
+ module.slots.append((self.prefix, rows))
+
+ def set_weight(self, weight: float) -> None:
+ import torch
+ with torch.no_grad():
+ getattr(self.wrapper, self.prefix + "_scale").fill_(float(weight) * self.alpha_scale)
+
+ def remove(self) -> None:
+ wrapper = self.wrapper
+ wrapper.slots = [slot for slot in wrapper.slots if slot[0] != self.prefix]
+ for suffix in ("_w1", "_w2", "_scale"):
+ wrapper._buffers.pop(self.prefix + suffix, None)
+ if not wrapper.slots:
+ parent, attr = _parent_of(self.root, wrapper)
+ setattr(parent, attr, wrapper.base_layer)
+
+
+def _kron_factors(parts: dict) -> tuple[Any, Any, float]:
+ """``(w1, w2, alpha_scale)`` like the reference runtime: a factor stored as a low-rank pair is its product, and
+ ``alpha / rank`` applies only when one is (rank = that pair's inner size)."""
+ import torch
+
+ dim = None
+ w1 = parts.get("lokr_w1")
+ if w1 is None:
+ w1 = parts["lokr_w1_a"].float() @ parts["lokr_w1_b"].float()
+ dim = int(parts["lokr_w1_b"].shape[0])
+ w2 = parts.get("lokr_w2")
+ if w2 is None:
+ if "lokr_t2" in parts:
+ raise ValueError("a LoKR convolution (lokr_t2) cannot apply to a Linear layer")
+ w2 = parts["lokr_w2_a"].float() @ parts["lokr_w2_b"].float()
+ dim = int(parts["lokr_w2_b"].shape[0])
+ if w1.dim() != 2 or w2.dim() != 2:
+ raise ValueError("only Linear LoKR modules are supported")
+ if "dora_scale" in parts:
+ raise ValueError("LoKR modules trained with DoRA (dora_scale) are not supported")
+ alpha = parts.get("alpha")
+ scale = float(alpha) / dim if alpha is not None and dim else 1.0
+ return w1.to(torch.float32), w2.to(torch.float32), scale
+
+
+def _device_of(module: Any) -> Any:
+ weight = getattr(module, "weight", None)
+ return weight.device if weight is not None and weight.device.type != "meta" else None
+
+
+def _dtype_of(module: Any) -> Any:
+ """The compute dtype of a layer: a quantized weight reports its storage dtype, so fall back to its bias or bf16."""
+ import torch
+
+ for name in ("weight", "bias"):
+ tensor = getattr(module, name, None)
+ if tensor is not None and tensor.is_floating_point() and tensor.element_size() >= 2:
+ return tensor.dtype
+ return torch.bfloat16
+
+
+def _base_layer(module: Any) -> Any:
+ get_base = getattr(module, "get_base_layer", None)
+ return get_base() if callable(get_base) else module
+
+
+def _lokr_targets(
+ pipe: Any, stem: str, family: Optional[str]
+) -> list[tuple[Any, Any, Optional[tuple[int, int]]]]:
+ key = _strip(stem)
+ if family == "krea-2":
+ from .diffusion_single_file_converters import krea2_checkpoint_to_diffusers
+ name = next(iter(krea2_checkpoint_to_diffusers({key + ".weight": None})))[: -len(".weight")]
+ return [(pipe.transformer, pipe.transformer.get_submodule(name), None)]
+ if family == "moss-image":
+ root = pipe.transformer
+ match = re.match(r"^(.*\.proj\.)qkv$", key)
+ if match is None:
+ return [(root, root.get_submodule(key), None)]
+ modules = [root.get_submodule(match.group(1) + n) for n in ("q", "k", "v")]
+ width = int(_base_layer(modules[0]).weight.shape[0])
+ return [(root, m, (i * width, (i + 1) * width)) for i, m in enumerate(modules)]
+ if family == "anima":
+ from diffusers.loaders.lora_conversion_utils import (
+ _convert_non_diffusers_anima_lora_to_diffusers,
+ )
+
+ name = next(
+ iter(
+ _convert_non_diffusers_anima_lora_to_diffusers(
+ {"diffusion_model." + key + ".x": None}
+ )
+ )
+ )
+ component, _, path = name[: -len(".x")].partition(".")
+ root = getattr(pipe, component)
+ return [(root, root.get_submodule(path), None)]
+ raise ValueError(f"LoKR adapters are not supported for {family or 'this model'}")
+
+
+def _attach_lokr(pipe: Any, name: str, lokr: dict, family: Optional[str]) -> int:
+ from torch import nn
+
+ adapters: list[KronAdapter] = []
+ try:
+ for stem, parts in lokr.items():
+ w1, w2, scale = _kron_factors(parts)
+ for root, module, rows in _lokr_targets(pipe, stem, family):
+ base = _base_layer(module)
+ if isinstance(base, nn.Embedding): # 2-D like a Linear, but its input is token ids
+ raise ValueError(
+ f"LoKR module {_strip(stem)} targets an embedding table; only Linear LoKR modules are supported"
+ )
+ out_f, in_f = (
+ (int(d) for d in base.weight.shape) if base.weight.dim() == 2 else (0, 0)
+ )
+ full = int(w1.shape[0] * w2.shape[0])
+ width = rows[1] - rows[0] if rows else full
+ if in_f != int(w1.shape[1] * w2.shape[1]) or out_f != width:
+ raise ValueError(
+ f"LoKR module {_strip(stem)} is {full} x {int(w1.shape[1] * w2.shape[1])}, "
+ f"but the layer it targets is {out_f} x {in_f}"
+ )
+ adapters.append(KronAdapter(root, module, w1, w2, scale, rows))
+ except Exception:
+ for adapter in adapters:
+ adapter.remove()
+ raise
+ registry = pipe.__dict__.setdefault(KRON_ATTR, {})
+ registry[name] = adapters
+ return len(adapters)
+
+
+def _resolve_kohya_stem(pipe: Any, stem: str, family: Optional[str]) -> str:
+ """The one dotted path, of every way to re-split ``stem``'s underscores, that names a layer of this model."""
+ tokens = stem.split("_")
+ if len(tokens) > 16:
+ raise ValueError(f"LoRA key {stem} names no layer of this model")
+ found = []
+ for mask in range(1 << (len(tokens) - 1)):
+ path = tokens[0] + "".join(
+ ("." if mask >> i & 1 else "_") + token for i, token in enumerate(tokens[1:])
+ )
+ try:
+ _lokr_targets(pipe, path, family)
+ except Exception: # noqa: BLE001 -- this split names no layer
+ continue
+ found.append(path)
+ if len(found) != 1:
+ raise ValueError(f"LoRA key {stem} names no single layer of this model")
+ return found[0]
+
+
+def _dotted_kohya_keys(pipe: Any, state: dict, family: Optional[str]) -> dict:
+ out: dict = {}
+ paths: dict = {}
+ for key, value in state.items():
+ prefix = next((p for p in _KOHYA_PREFIXES if key.startswith(p)), None)
+ if prefix is None:
+ out[key] = value
+ continue
+ stem, _, suffix = key[len(prefix) :].partition(".")
+ if stem not in paths:
+ paths[stem] = _resolve_kohya_stem(pipe, stem, family)
+ out[f"diffusion_model.{paths[stem]}.{suffix}"] = value
+ return out
+
+
+def _adapter_loaded(pipe: Any, name: str) -> bool:
+ return any(name in names for names in (pipe.get_list_adapters() or {}).values())
+
+
+def load_adapter(pipe: Any, path: str, name: str, family: Optional[str]) -> None:
+ """One file under ``name``: plain LoRA via ``pipe.load_lora_weights``, LoKR modules as ``KronLinear`` slots."""
+ if family not in NORMALIZED_FAMILIES:
+ pipe.load_lora_weights(path, adapter_name = name)
+ return
+ raw = read_lora_file(path)
+ lokr, rest = split_lokr(_dotted_kohya_keys(pipe, raw, family))
+ plain = normalize_lora(rest, family)
+ if not any(".lora_" in k for k in plain):
+ plain = {}
+ if plain:
+ # A file needing no rewrite loads by path: diffusers reads lora_alpha / rank from its metadata, not a dict.
+ unchanged = not lokr and plain.keys() == raw.keys() and all(plain[k] is raw[k] for k in raw)
+ # PEFT injects only into bare Linears: unwrap LoKR while it runs, then rewrap whatever sits there now.
+ wrappers = _unwrap_kron(pipe)
+ try:
+ pipe.load_lora_weights(path if unchanged else plain, adapter_name = name)
+ finally:
+ _rewrap_kron(wrappers)
+ if wrappers:
+ _refresh_group_offload(_kron_roots(pipe))
+ # diffusers drops keys outside the components it loads into without an error.
+ if not _adapter_loaded(pipe, name):
+ raise ValueError(f"{name}: none of its LoRA keys name a layer of this model")
+ if lokr:
+ attached = _attach_lokr(pipe, name, lokr, family)
+ _refresh_group_offload(_kron_roots(pipe))
+ if not plain:
+ pipe.__dict__.setdefault(KRON_ONLY_ATTR, set()).add(name)
+ logger.info("diffusion.lora: %s applies %d LoKR module(s)", name, attached)
+ if not plain and not lokr:
+ raise ValueError(f"{name} holds no LoRA or LoKR weights this loader recognises")
+
+
+def _kron_roots(pipe: Any) -> list:
+ roots = {
+ id(adapter.root): adapter.root
+ for adapters in pipe.__dict__.get(KRON_ATTR, {}).values()
+ for adapter in adapters
+ }
+ return list(roots.values())
+
+
+def _refresh_group_offload(roots: list) -> None:
+ """Re-apply group offloading on ``roots`` (as diffusers does after PEFT injection): groups list their tensors when
+ applied, so LoKR buffers added or dropped since would be missed."""
+ if not roots:
+ return
+ from diffusers.hooks.group_offloading import _maybe_remove_and_reapply_group_offloading
+
+ for root in roots:
+ _maybe_remove_and_reapply_group_offloading(root)
+
+
+def _unwrap_kron(pipe: Any) -> list:
+ out = []
+ seen: set = set()
+ for adapters in pipe.__dict__.get(KRON_ATTR, {}).values():
+ for adapter in adapters:
+ wrapper = adapter.wrapper
+ if id(wrapper) in seen:
+ continue
+ seen.add(id(wrapper))
+ parent, attr = _parent_of(adapter.root, wrapper)
+ setattr(parent, attr, wrapper.base_layer)
+ out.append((parent, attr, wrapper))
+ return out
+
+
+def _rewrap_kron(wrappers: list) -> None:
+ for parent, attr, wrapper in wrappers:
+ wrapper.base_layer = getattr(parent, attr)
+ setattr(parent, attr, wrapper)
+
+
+def kron_names(pipe: Any) -> tuple[str, ...]:
+ return tuple(pipe.__dict__.get(KRON_ATTR, {}))
+
+
+def set_adapter_weights(pipe: Any, names: list, weights: list) -> None:
+ """``pipe.set_adapters`` for both kinds; a LoKR-only name has no PEFT adapter to set."""
+ set_kron_weights(pipe, dict(zip(names, weights)))
+ kron_only = pipe.__dict__.get(KRON_ONLY_ATTR, set())
+ pairs = [(n, w) for n, w in zip(names, weights) if n not in kron_only]
+ if pairs or not kron_only:
+ pipe.set_adapters([n for n, _w in pairs], adapter_weights = [w for _n, w in pairs])
+
+
+def kron_only_selection(pipe: Any) -> bool:
+ names = [entry[0] for entry in getattr(pipe, "_unsloth_loras", ()) or ()]
+ kron_only = pipe.__dict__.get(KRON_ONLY_ATTR, set())
+ return bool(names) and all(name in kron_only for name in names)
+
+
+def set_kron_weights(pipe: Any, weights: dict) -> None:
+ """Every LoKR adapter's weight (name -> weight; an omitted name is 0, as a pinned PEFT scale is)."""
+ for adapter_name, adapters in pipe.__dict__.get(KRON_ATTR, {}).items():
+ for adapter in adapters:
+ adapter.set_weight(float(weights.get(adapter_name, 0.0)))
+
+
+def unload_adapters(pipe: Any) -> None:
+ roots = _kron_roots(pipe)
+ for adapters in pipe.__dict__.pop(KRON_ATTR, {}).values():
+ for adapter in adapters:
+ adapter.remove()
+ pipe.__dict__.pop(KRON_ONLY_ATTR, None)
+ pipe.unload_lora_weights()
+ _refresh_group_offload(roots)
diff --git a/studio/backend/core/inference/diffusion_moss_image.py b/studio/backend/core/inference/diffusion_moss_image.py
index 2a907729b14..df405e5470d 100644
--- a/studio/backend/core/inference/diffusion_moss_image.py
+++ b/studio/backend/core/inference/diffusion_moss_image.py
@@ -334,6 +334,7 @@ def _model_bases():
_ModelMixin, _ConfigMixin = _model_bases()
+from diffusers.loaders import PeftAdapterMixin as _PeftAdapterMixin # noqa: E402 (LoRA: load_lora_adapter)
from diffusers.configuration_utils import register_to_config # noqa: E402
@@ -346,7 +347,7 @@ def _float64_device(device):
return torch.device("cpu")
-class MOSSImageTransformer(_ModelMixin, _ConfigMixin):
+class MOSSImageTransformer(_ModelMixin, _ConfigMixin, _PeftAdapterMixin):
"""MOSS-Image DiT: a text refiner, then 42 single-stream joint blocks over [image ; text] sharing one AdaLN.
Each sample runs at its true text length, text refinement included, so padded tokens never reach attention."""
@@ -688,6 +689,47 @@ class MOSSImagePipeline(DiffusionPipeline):
def interrupt(self) -> bool:
return self._interrupt
+ # LoRA on the transformer only; diffusion_lora_files folds alpha and splits fused qkv first.
+ def load_lora_weights(
+ self,
+ state_dict,
+ adapter_name: Optional[str] = None,
+ **_kwargs,
+ ):
+ metadata = None
+ if not isinstance(state_dict, dict):
+ from diffusers.utils.state_dict_utils import _load_sft_state_dict_metadata
+ from safetensors.torch import load_file
+
+ # lora_alpha / rank of a diffusers-saved adapter live only in the file's metadata.
+ metadata = _load_sft_state_dict_metadata(str(state_dict))
+ state_dict = load_file(str(state_dict))
+ converted = {}
+ for key, value in state_dict.items():
+ for prefix in ("diffusion_model.", "model.diffusion_model.", "transformer."):
+ if key.startswith(prefix):
+ key = key[len(prefix) :]
+ break
+ converted["transformer." + key] = value
+ self.transformer.load_lora_adapter(
+ converted, adapter_name = adapter_name, prefix = "transformer", metadata = metadata
+ )
+
+ def set_adapters(
+ self,
+ adapter_names,
+ adapter_weights = None,
+ ):
+ self.transformer.set_adapters(adapter_names, weights = adapter_weights)
+
+ def get_list_adapters(self) -> dict:
+ config = getattr(self.transformer, "peft_config", None) or {}
+ return {"transformer": list(config)} if config else {}
+
+ def unload_lora_weights(self):
+ if getattr(self.transformer, "peft_config", None):
+ self.transformer.unload_lora()
+
@torch.no_grad()
def encode_prompt(
self,
diff --git a/studio/backend/tests/test_diffusion_lora.py b/studio/backend/tests/test_diffusion_lora.py
index 66807d58160..c4c03edd0f8 100644
--- a/studio/backend/tests/test_diffusion_lora.py
+++ b/studio/backend/tests/test_diffusion_lora.py
@@ -157,8 +157,11 @@ def test_lora_unavailable_reason_names_the_cause():
assert "NVFP4" in dl.lora_unavailable_reason(
engine = "diffusers", family = "flux.1", model_kind = "pipeline", transformer_quant = "nvfp4"
)
- assert "no LoRA loader" in dl.lora_unavailable_reason(
- engine = "diffusers", family = "moss-image", model_kind = "pipeline", transformer_quant = None
+ assert (
+ dl.lora_unavailable_reason(
+ engine = "diffusers", family = "moss-image", model_kind = "pipeline", transformer_quant = None
+ )
+ is None
)
assert "Speed" in dl.lora_unavailable_reason(
engine = "diffusers",
@@ -507,6 +510,26 @@ def test_baked_quant_pipe_disables_any_subset_of_its_adapters(monkeypatch):
assert pipe.active == (["style", "style_2"], [0.7, 1.0])
+def test_a_declined_quantize_keeps_the_dense_lora_path(monkeypatch):
+ import threading
+
+ monkeypatch.setattr(
+ dl,
+ "resolve_specs",
+ lambda specs, **_: [
+ dl.ResolvedLora(i, dl.sanitize_alias(i), f"/{i}.safetensors", "safetensors", w)
+ for i, w in specs
+ ],
+ )
+ b = _backend()
+ pipe = _FakePipe()
+ pipe._unsloth_loras = b._resolve_lora_set([("styleA", 1.0)], family = "flux.1", hf_token = None)
+ pipe._unsloth_loras_baked = True
+ b._apply_loras(_fake_state(pipe), [("styleB", 1.0)], threading.Event())
+ assert [n for _p, n in pipe.loaded] == ["styleB"]
+ assert pipe.active == (["styleB"], [1.0])
+
+
@pytest.mark.parametrize("mode, refused", [("group", True), ("resident", False), ("model", False)])
def test_baked_weights_stay_fixed_under_a_block_streamed_graph(monkeypatch, mode, refused):
# Resetting a block-streamed graph for new LoRA scales left every later generate failing. Refuse the change.
@@ -594,16 +617,29 @@ def test_scan_local_tolerates_bad_sidecar(tmp_path, monkeypatch):
assert entry.families == () and entry.weight_default == 1.0
-def test_moss_image_has_no_lora_loader_so_lora_is_gated_off(monkeypatch):
+def test_moss_image_has_a_lora_loader():
+ pytest.importorskip("torch")
+ pytest.importorskip("diffusers")
+ from core.inference.diffusion_moss_image import MOSSImagePipeline, MOSSImageTransformer
+
+ for method in ("load_lora_weights", "set_adapters", "unload_lora_weights", "get_list_adapters"):
+ assert hasattr(MOSSImagePipeline, method)
+ assert hasattr(MOSSImageTransformer, "load_lora_adapter")
+ assert dl.supports_lora(
+ engine = "diffusers", family = "moss-image", model_kind = "pipeline", transformer_quant = None
+ )
+
+
+def test_a_family_without_a_lora_loader_is_gated_off(monkeypatch):
+ """The gate for an in-tree pipeline with no LoRA loader, exercised on MOSS-Image as if it still had none."""
import threading
pytest.importorskip("torch")
pytest.importorskip("diffusers")
from core.inference.diffusion_families import detect_family
- from core.inference.diffusion_moss_image import MOSSImagePipeline
- assert not hasattr(MOSSImagePipeline, "load_lora_weights")
+ monkeypatch.setattr(dl, "_DIFFUSERS_LORA_UNSUPPORTED_FAMILIES", ("moss-image",))
for quant in (None, "int8", "fp8"):
assert not dl.supports_lora(
engine = "diffusers", family = "moss-image", model_kind = "pipeline", transformer_quant = quant
@@ -1056,3 +1092,47 @@ def test_diffusers_weight_change_falls_back_to_reload_when_set_adapters_fails(
assert pipe.unloaded == 1
assert pipe.active == (["styleA"], [0.5])
assert [(n, w) for n, _p, w in pipe._unsloth_loras] == [("styleA", 0.5)]
+
+
[email protected]("baked", [True, False])
+def test_a_lokr_only_weight_change_keeps_the_cuda_graphs(monkeypatch, tmp_path, baked):
+ """A LoKR-only strength change (or disable) writes scale tensors in place: graphs kept, CFG replica dropped."""
+ import threading
+
+ from core.inference import diffusion as diffusion_mod
+ from core.inference import diffusion_lora_files as lf
+
+ _real_file_specs(monkeypatch, tmp_path)
+ resets: list = []
+ invalidated: list = []
+ monkeypatch.setattr(
+ diffusion_mod.cuda_graph, "reset_all", lambda handles: resets.append(handles)
+ )
+ monkeypatch.setattr(
+ diffusion_mod, "invalidate_cfg_replica", lambda pipe, why, log = None: invalidated.append(why)
+ )
+ # The scale mirror copies PEFT scales only, so even a mirror that lines up must not keep the replica.
+ monkeypatch.setattr(
+ diffusion_mod, "mirror_cfg_replica_adapter_scales", lambda pipe, log = None: True
+ )
+ torch = pytest.importorskip("torch")
+ block = torch.nn.ModuleDict({"q": torch.nn.Linear(24, 36, bias = False)})
+ adapter = lf.KronAdapter(block, block["q"], torch.randn(4, 3), torch.randn(9, 8), 1.0)
+ pipe = _FakePipe()
+ pipe.__dict__[lf.KRON_ATTR] = {"styleA": [adapter]}
+ pipe.__dict__[lf.KRON_ONLY_ATTR] = {"styleA"}
+ b = _backend()
+ state = _fake_state(pipe, quant = "int8" if baked else None)
+ pipe._unsloth_loras = b._resolve_lora_set([("styleA", 1.0)], family = "flux.1", hf_token = None)
+ pipe._unsloth_loras_baked = baked
+ if not baked:
+ monkeypatch.setattr(diffusion_mod, "_lora_weights_only_change", lambda *a: True)
+ monkeypatch.setattr(diffusion_mod, "_lora_load_block_reason", lambda pipe: None)
+
+ b._apply_loras(state, [("styleA", 0.4)], threading.Event())
+ assert float(getattr(block["q"], adapter.prefix + "_scale")) == pytest.approx(0.4)
+ if baked:
+ b._apply_loras(state, [], threading.Event()) # disabling every adapter
+ assert float(getattr(block["q"], adapter.prefix + "_scale")) == 0.0
+ assert resets == []
+ assert invalidated
diff --git a/studio/backend/tests/test_diffusion_lora_files.py b/studio/backend/tests/test_diffusion_lora_files.py
new file mode 100644
index 00000000000..11824dbbc7c
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_lora_files.py
@@ -0,0 +1,442 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""LoRA and LyCORIS LoKR files: key normalization and the Kronecker adapter."""
+
+from __future__ import annotations
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+from core.inference import diffusion_lora_files as lf # noqa: E402
+
+
+def test_alpha_folds_into_lora_b_and_down_up_rename():
+ a, b = torch.randn(4, 8), torch.randn(6, 4)
+ out = lf.normalize_lora(
+ {
+ "diffusion_model.blocks.0.attn.wq.lora_down.weight": a,
+ "diffusion_model.blocks.0.attn.wq.lora_up.weight": b,
+ "diffusion_model.blocks.0.attn.wq.alpha": torch.tensor(2.0),
+ "diffusion_model.blocks.0.attn.wk.alpha": torch.tensor(
+ 1.0
+ ), # an orphan alpha is dropped
+ },
+ "krea-2",
+ )
+ assert sorted(out) == [
+ "diffusion_model.blocks.0.attn.wq.lora_A.weight",
+ "diffusion_model.blocks.0.attn.wq.lora_B.weight",
+ ]
+ assert torch.allclose(out["diffusion_model.blocks.0.attn.wq.lora_B.weight"], b * 0.5)
+
+
+def test_anima_embedding_pair_becomes_a_peft_embedding_lora():
+ a, b = (
+ torch.randn(3, 5),
+ torch.randn(11, 3),
+ ) # ComfyUI: delta = B @ A on the [vocab 11, dim 5] table
+ out = lf.normalize_lora(
+ {
+ "diffusion_model.llm_adapter.embed.lora_A.weight": a,
+ "diffusion_model.llm_adapter.embed.lora_B.weight": b,
+ },
+ "anima",
+ )
+ ea = out["diffusion_model.llm_adapter.embed.lora_embedding_A"]
+ eb = out["diffusion_model.llm_adapter.embed.lora_embedding_B"]
+ # PEFT's embedding delta is (B' @ A').T
+ assert torch.allclose((eb @ ea).T, b @ a)
+
+
+def test_moss_fused_qkv_lora_splits_rows_and_shares_a():
+ a, b = torch.randn(4, 6), torch.randn(9, 4)
+ out = lf.normalize_lora(
+ {
+ "diffusion_model.layers.0.proj.qkv.lora_A.weight": a,
+ "diffusion_model.layers.0.proj.qkv.lora_B.weight": b,
+ },
+ "moss-image",
+ )
+ for i, name in enumerate(("q", "k", "v")):
+ assert torch.equal(out[f"diffusion_model.layers.0.proj.{name}.lora_A.weight"], a)
+ assert torch.equal(
+ out[f"diffusion_model.layers.0.proj.{name}.lora_B.weight"], b[i * 3 : (i + 1) * 3]
+ )
+
+
+def test_split_lokr_keeps_alpha_with_its_module():
+ w1, w2 = torch.randn(2, 2), torch.randn(3, 3)
+ lokr, rest = lf.split_lokr(
+ {
+ "m.lokr_w1": w1,
+ "m.lokr_w2": w2,
+ "m.alpha": torch.tensor(4.0),
+ "n.lora_A.weight": torch.randn(1, 6),
+ "n.alpha": torch.tensor(1.0),
+ }
+ )
+ assert set(lokr["m"]) == {"lokr_w1", "lokr_w2", "alpha"}
+ assert set(rest) == {"n.lora_A.weight", "n.alpha"}
+
+
+def test_kron_factors_scale_like_comfyui():
+ w1, w2 = torch.randn(2, 2), torch.randn(3, 3)
+ # Full factors: alpha is ignored (scale 1).
+ assert lf._kron_factors({"lokr_w1": w1, "lokr_w2": w2, "alpha": torch.tensor(8.0)})[2] == 1.0
+ # A low-rank factor: alpha / its rank.
+ _, _, scale = lf._kron_factors(
+ {
+ "lokr_w1": w1,
+ "lokr_w2_a": torch.randn(3, 2),
+ "lokr_w2_b": torch.randn(2, 3),
+ "alpha": torch.tensor(1.0),
+ }
+ )
+ assert scale == 0.5
+
+
+class _Block(torch.nn.Module):
+ def __init__(self, out_features = 36):
+ super().__init__()
+ self.proj = torch.nn.ModuleDict({"q": torch.nn.Linear(24, out_features, bias = False)})
+
+ def forward(self, x):
+ return self.proj["q"](x)
+
+
+def test_kron_adapter_matches_the_dense_delta_and_restores_the_layer():
+ torch.manual_seed(0)
+ block = _Block().double()
+ base = block.proj["q"]
+ w1, w2 = torch.randn(4, 3).double(), torch.randn(9, 8).double()
+ first = lf.KronAdapter(block, base, w1, w2, 1.0)
+ second = lf.KronAdapter(block, block.proj["q"], w1, w2, 1.0)
+ first.set_weight(0.7)
+ second.set_weight(0.3)
+ x = torch.randn(2, 5, 24, dtype = torch.float64)
+ expected = x @ (base.weight + torch.kron(w1, w2)).T
+ assert torch.allclose(block(x), expected, atol = 1e-10)
+ assert block.proj["q"].weight is base.weight
+ first.remove()
+ second.remove()
+ assert block.proj["q"] is base
+
+
+def test_kron_adapter_row_slice_for_a_fused_qkv_factor():
+ torch.manual_seed(1)
+ block = _Block(out_features = 12).double()
+ w1, w2 = (
+ torch.randn(4, 3).double(),
+ torch.randn(9, 8).double(),
+ ) # 36 rows, this layer takes rows 12..24
+ lf.KronAdapter(block, block.proj["q"], w1, w2, 1.0, rows = (12, 24))
+ x = torch.randn(3, 24, dtype = torch.float64)
+ base = block.proj["q"].base_layer
+ assert torch.allclose(block(x), x @ (base.weight + torch.kron(w1, w2)[12:24]).T, atol = 1e-10)
+
+
+def test_a_failed_lokr_allocation_leaves_the_layer_alone():
+ class _OutOfMemory:
+ def to(self, **_kwargs):
+ raise RuntimeError("CUDA out of memory")
+
+ block = _Block()
+ base = block.proj["q"]
+ with pytest.raises(RuntimeError, match = "out of memory"):
+ lf.KronAdapter(block, base, torch.randn(4, 3), _OutOfMemory(), 1.0)
+ assert block.proj["q"] is base
+
+
+def test_the_moss_loader_keeps_the_adapter_metadata(tmp_path):
+ pytest.importorskip("diffusers")
+ import json
+ from types import SimpleNamespace
+
+ from safetensors.torch import save_file
+
+ from core.inference.diffusion_moss_image import MOSSImagePipeline
+
+ path = tmp_path / "moss.safetensors"
+ meta = {"transformer.r": 2, "transformer.lora_alpha": 8}
+ save_file(
+ {
+ "transformer.layers.0.proj.q.lora_A.weight": torch.randn(2, 4),
+ "transformer.layers.0.proj.q.lora_B.weight": torch.randn(4, 2),
+ },
+ str(path),
+ metadata = {"lora_adapter_metadata": json.dumps(meta)},
+ )
+ calls = []
+ pipe = SimpleNamespace(
+ transformer = SimpleNamespace(load_lora_adapter = lambda sd, **kw: calls.append((sd, kw)))
+ )
+ MOSSImagePipeline.load_lora_weights(pipe, str(path), adapter_name = "m")
+ ((state, kwargs),) = calls
+ assert sorted(state) == [
+ "transformer.layers.0.proj.q.lora_A.weight",
+ "transformer.layers.0.proj.q.lora_B.weight",
+ ]
+ assert kwargs["metadata"] == meta
+
+
+def test_peft_loads_around_an_existing_lokr_wrapper():
+ """A LoRA added after a LoKR on one layer: PEFT injects into the bare Linear, then the wrapper goes back on."""
+ pytest.importorskip("peft")
+ from core.inference.diffusion_torchao_patches import install_peft_torchao_dispatch_guard
+
+ install_peft_torchao_dispatch_guard() # as every diffusion entry point does
+ block = _Block().double()
+
+ class _Pipe:
+ def __init__(self, transformer):
+ self.transformer = transformer
+
+ def load_lora_weights(
+ self,
+ _state,
+ adapter_name = None,
+ ):
+ from peft import LoraConfig, inject_adapter_in_model
+ inject_adapter_in_model(
+ LoraConfig(r = 2, target_modules = ["q"]), self.transformer, adapter_name
+ )
+
+ pipe = _Pipe(block)
+ adapter = lf.KronAdapter(
+ block, block.proj["q"], torch.randn(4, 3).double(), torch.randn(9, 8).double(), 1.0
+ )
+ pipe.__dict__[lf.KRON_ATTR] = {"k": [adapter]}
+ wrappers = lf._unwrap_kron(pipe)
+ pipe.load_lora_weights({}, adapter_name = "l")
+ lf._rewrap_kron(wrappers)
+ wrapper = block.proj["q"]
+ assert isinstance(wrapper, lf.kron_linear_class())
+ assert type(wrapper.base_layer).__name__ == "Linear" and hasattr(wrapper.base_layer, "lora_A")
+
+
+def test_an_embedding_lokr_is_refused_at_load(tmp_path):
+ """A LoKR on Anima's ``llm_adapter.embed`` table (token ids, not features) fails at load, not at render."""
+ conversion = pytest.importorskip("diffusers.loaders.lora_conversion_utils")
+ if not hasattr(conversion, "_convert_non_diffusers_anima_lora_to_diffusers"):
+ pytest.skip("this diffusers has no Anima LoRA converter")
+ from safetensors.torch import save_file
+
+ class _Conditioner(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.embed = torch.nn.Embedding(12, 6)
+
+ class _Pipe:
+ def __init__(self):
+ self.text_conditioner = _Conditioner()
+
+ pipe = _Pipe()
+ embed = pipe.text_conditioner.embed
+ path = tmp_path / "lokr.safetensors"
+ # A [12, 6] table: w1 [3, 2], w2 [4, 3].
+ save_file(
+ {
+ "diffusion_model.llm_adapter.embed.lokr_w1": torch.randn(3, 2),
+ "diffusion_model.llm_adapter.embed.lokr_w2": torch.randn(4, 3),
+ },
+ path,
+ )
+ with pytest.raises(ValueError, match = "embedding"):
+ lf.load_adapter(pipe, str(path), "k", "anima")
+ assert pipe.text_conditioner.embed is embed
+ assert lf.kron_names(pipe) == ()
+
+
+class _Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList([_Block(), _Block(24)])
+
+
+class _RecordingPipe:
+ def __init__(self, loads = True):
+ self.transformer = _Net()
+ self.loads = loads
+ self.loaded: dict = {}
+
+ def load_lora_weights(
+ self,
+ state,
+ adapter_name = None,
+ ):
+ if self.loads:
+ self.loaded[adapter_name] = dict(state)
+
+ def get_list_adapters(self):
+ return {"transformer": list(self.loaded)} if self.loaded else {}
+
+
+def test_kohya_keys_resolve_to_the_layer_they_name(tmp_path, monkeypatch):
+ from safetensors.torch import save_file
+
+ monkeypatch.setattr(lf, "_refresh_group_offload", lambda roots: None) # no diffusers needed
+
+ a, b = torch.randn(2, 24), torch.randn(24, 2)
+ path = tmp_path / "kohya.safetensors"
+ save_file(
+ {
+ "lora_unet_blocks_1_proj_q.lora_down.weight": a,
+ "lora_unet_blocks_1_proj_q.lora_up.weight": b,
+ "lora_unet_blocks_1_proj_q.alpha": torch.tensor(1.0),
+ "lycoris_blocks_0_proj_q.lokr_w1": torch.randn(4, 3),
+ "lycoris_blocks_0_proj_q.lokr_w2": torch.randn(9, 8),
+ },
+ path,
+ )
+ pipe = _RecordingPipe()
+ lf.load_adapter(pipe, str(path), "k", "moss-image")
+ loaded = pipe.loaded["k"]
+ assert sorted(loaded) == [
+ "diffusion_model.blocks.1.proj.q.lora_A.weight",
+ "diffusion_model.blocks.1.proj.q.lora_B.weight",
+ ]
+ assert torch.allclose(loaded["diffusion_model.blocks.1.proj.q.lora_B.weight"], b * 0.5)
+ assert isinstance(pipe.transformer.blocks[0].proj["q"], lf.kron_linear_class())
+
+
+def test_a_file_needing_no_rewrite_loads_by_path(tmp_path):
+ from safetensors.torch import save_file
+
+ diffusers_format = tmp_path / "diffusers.safetensors"
+ save_file(
+ {
+ "transformer.blocks.1.proj.q.lora_A.weight": torch.randn(2, 24),
+ "transformer.blocks.1.proj.q.lora_B.weight": torch.randn(24, 2),
+ },
+ diffusers_format,
+ metadata = {"lora_adapter_metadata": '{"r": 2, "lora_alpha": 8}'},
+ )
+ pipe = _RecordingPipe()
+ received = []
+ pipe.load_lora_weights = lambda source, adapter_name = None: (
+ received.append(source),
+ pipe.loaded.setdefault(adapter_name, {}),
+ )
+ lf.load_adapter(pipe, str(diffusers_format), "d", "krea-2")
+ assert received == [str(diffusers_format)]
+ reference_format = tmp_path / "reference.safetensors"
+ save_file(
+ {
+ "diffusion_model.blocks.1.proj.q.lora_down.weight": torch.randn(2, 24),
+ "diffusion_model.blocks.1.proj.q.lora_up.weight": torch.randn(24, 2),
+ },
+ reference_format,
+ )
+ lf.load_adapter(pipe, str(reference_format), "r", "krea-2")
+ assert isinstance(received[-1], dict)
+
+
+def test_a_lora_that_loads_into_no_layer_is_refused(tmp_path):
+ from safetensors.torch import save_file
+
+ path = tmp_path / "te_only.safetensors"
+ save_file(
+ {
+ "lora_te_layers_0_mlp_up.lora_down.weight": torch.randn(2, 8),
+ "lora_te_layers_0_mlp_up.lora_up.weight": torch.randn(8, 2),
+ },
+ path,
+ )
+ with pytest.raises(ValueError, match = "none of its LoRA keys"):
+ lf.load_adapter(_RecordingPipe(loads = False), str(path), "k", "moss-image")
+
+
+def test_a_dora_lokr_is_refused(tmp_path):
+ from safetensors.torch import save_file
+
+ path = tmp_path / "dora.safetensors"
+ save_file(
+ {
+ "diffusion_model.blocks.0.proj.q.lokr_w1": torch.randn(4, 3),
+ "diffusion_model.blocks.0.proj.q.lokr_w2": torch.randn(9, 8),
+ "diffusion_model.blocks.0.proj.q.dora_scale": torch.ones(1, 24),
+ },
+ path,
+ )
+ pipe = _RecordingPipe()
+ with pytest.raises(ValueError, match = "DoRA"):
+ lf.load_adapter(pipe, str(path), "k", "moss-image")
+ assert lf.kron_names(pipe) == ()
+
+
+def test_compiled_block_reads_a_new_lokr_weight():
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ block = _Block().cuda().bfloat16()
+ adapter = lf.KronAdapter(block, block.proj["q"], torch.randn(4, 3), torch.randn(9, 8), 1.0)
+ block.compile(fullgraph = True)
+ x = torch.randn(2, 24, device = "cuda", dtype = torch.bfloat16)
+ on = block(x)
+ adapter.set_weight(0.0)
+ off = block(x)
+ assert (on - off).abs().max() > 0
+ assert torch.allclose(off, x @ block.proj["q"].weight.T)
+
+
+def test_moss_and_anima_have_a_lora_loader():
+ from core.inference.diffusion_lora import diffusers_lora_family_supported
+ assert diffusers_lora_family_supported("moss-image")
+ assert diffusers_lora_family_supported("anima")
+
+
[email protected]("use_stream", [False, True])
+def test_lokr_attached_under_group_offload_moves_with_its_group(tmp_path, use_stream):
+ """A LoKR attached at generate time to a group-offloaded transformer: the groups are rebuilt, so the factors onload
+ with their block, and again on unload."""
+ if not torch.cuda.is_available():
+ pytest.skip("needs CUDA")
+ from diffusers.hooks import apply_group_offloading
+ from safetensors.torch import save_file
+
+ torch.manual_seed(0)
+
+ class _Net(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.blocks = torch.nn.ModuleList([_Block(), _Block(24)])
+
+ def forward(self, x):
+ return self.blocks[0](self.blocks[1](x))
+
+ class _Pipe:
+ def __init__(self, transformer):
+ self.transformer = transformer
+
+ def unload_lora_weights(self):
+ pass
+
+ net = _Net().bfloat16()
+ apply_group_offloading(
+ net,
+ onload_device = torch.device("cuda"),
+ offload_device = torch.device("cpu"),
+ offload_type = "block_level",
+ num_blocks_per_group = 1,
+ use_stream = use_stream,
+ )
+ x = torch.randn(2, 24, device = "cuda", dtype = torch.bfloat16)
+ bare = net(x)
+ w1, w2 = torch.randn(4, 3), torch.randn(9, 8)
+ path = tmp_path / "lokr.safetensors"
+ save_file(
+ {
+ "diffusion_model.blocks.0.proj.q.lokr_w1": w1,
+ "diffusion_model.blocks.0.proj.q.lokr_w2": w2,
+ },
+ path,
+ )
+ pipe = _Pipe(net)
+ lf.load_adapter(pipe, str(path), "k", "moss-image")
+ out = net(x)
+ inner = net.blocks[1](x)
+ expected = bare + (inner @ torch.kron(w1, w2).to("cuda", torch.bfloat16).T)
+ assert torch.allclose(out.float(), expected.float(), atol = 0.25, rtol = 0.05)
+ lf.unload_adapters(pipe)
+ assert torch.equal(net(x), bare)