Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Z-Image-Turbo fused QKV GEMM (LoRA wrappers fall back to stock); UNSLOTH_DIFFUSION_ZIMAGE_FUSED=0"""
from __future__ import annotations
import functools
import os
import threading
from typing import Any, Optional
ZIMAGE_FUSED_ENV = "UNSLOTH_DIFFUSION_ZIMAGE_FUSED"
_MODULE = "diffusers.models.transformers.transformer_z_image"
_FINGERPRINTS = {"ZSingleStreamAttnProcessor.__call__": frozenset({"c87f9c27d7df695f"})}
_QKV_ATTR = "_unsloth_fused_qkv"
_LOCK = threading.Lock()
_STATE: dict = {}
# device index -> (even lanes, odd lanes) fma form of this card's complex multiply, or None
_FUSION: dict = {}
def zimage_fused_disabled() -> bool:
return (os.environ.get(ZIMAGE_FUSED_ENV) or "").strip().lower() in ("0", "off", "false", "no")
def _stock_digest(cls: Any) -> Optional[str]:
from .diffusion_qwenimage21_rope import _digest
return _digest(getattr(cls, "__call__", None))
def _real_rope(x: Any, freqs_cis: Any, fusion: tuple) -> Any:
"""``view_as_real(view_as_complex(x.float()) * freqs_cis.unsqueeze(2)).flatten(3).type_as(x)`` in real ops.
``x``: [B, S, H, D]; ``freqs_cis``: [B, S, D/2] complex64."""
import torch
xf = x.float()
f = torch.view_as_real(freqs_cis).unsqueeze(2)
cos = f[..., :1].expand(*f.shape[:-1], 2).flatten(-2)
sin = torch.stack([-f[..., 1], f[..., 1]], dim = -1).flatten(-2)
swapped = xf.unflatten(-1, (-1, 2)).flip(-1).flatten(-2)
on_x = torch.addcmul(swapped * sin, xf, cos) if "x" in fusion else None
on_s = torch.addcmul(xf * cos, swapped, sin) if "s" in fusion else None
if on_s is None:
out = on_x
elif on_x is None:
out = on_s
else:
even = torch.arange(x.shape[-1], device = x.device) % 2 == 0
out = torch.where(even, on_x, on_s) if fusion[0] == "x" else torch.where(even, on_s, on_x)
return out.type_as(x)
def _qkv_intact(attn: Any) -> Any:
"""The fused Linear, if ``to_q/to_k/to_v`` are still the exact modules it was built from."""
rec = attn.__dict__.get(_QKV_ATTR)
if rec is None:
return None
lin, q, k, v = rec
if attn.to_q is not q or attn.to_k is not k or attn.to_v is not v:
return None
if lin.weight.device != q.weight.device:
return None # a layer split moved the parts after the fusion; the stale fused copy is on another card
return lin
def _make_call(stock: Any) -> Any:
import torch
# bound here, not inside the traced __call__
from .diffusion_int8_fused import ensure_intmm_module, int8_linear
ensure_intmm_module() # resolved here, eagerly: an import under the trace is a graph break per Linear
def __call__(
self,
attn,
hidden_states,
encoder_hidden_states = None,
attention_mask = None,
freqs_cis = None,
):
if not torch.compiler.is_compiling():
return stock(
self, attn, hidden_states, encoder_hidden_states, attention_mask, freqs_cis
)
from diffusers.models.attention_dispatch import dispatch_attention_fn
fused = _qkv_intact(attn)
if fused is not None:
query, key, value = int8_linear(fused, hidden_states).chunk(3, dim = -1)
else:
query = attn.to_q(hidden_states)
key = attn.to_k(hidden_states)
value = attn.to_v(hidden_states)
query = query.unflatten(-1, (attn.heads, -1))
key = key.unflatten(-1, (attn.heads, -1))
value = value.unflatten(-1, (attn.heads, -1))
if attn.norm_q is not None:
query = attn.norm_q(query)
if attn.norm_k is not None:
key = attn.norm_k(key)
if freqs_cis is not None:
fusion = _FUSION.get(query.device.index) if query.is_cuda else None
if (
fusion is not None
and freqs_cis.dtype == torch.complex64
and query.shape[-1] % 2 == 0
):
query = _real_rope(query, freqs_cis, fusion)
key = _real_rope(key, freqs_cis, fusion)
else:
query = _stock_rope(query, freqs_cis)
key = _stock_rope(key, freqs_cis)
dtype = query.dtype
query, key = query.to(dtype), key.to(dtype)
if attention_mask is not None and attention_mask.ndim == 2:
attention_mask = attention_mask[:, None, None, :]
hidden_states = dispatch_attention_fn(
query,
key,
value,
attn_mask = attention_mask,
dropout_p = 0.0,
is_causal = False,
backend = self._attention_backend,
parallel_config = self._parallel_config,
)
hidden_states = hidden_states.flatten(2, 3).to(dtype)
output = int8_linear(attn.to_out[0], hidden_states)
if len(attn.to_out) > 1:
output = attn.to_out[1](output)
return output
functools.update_wrapper(__call__, stock, assigned = ("__name__", "__doc__"), updated = ())
__call__.__unsloth_zimage_fused__ = True
return __call__
def _stock_rope(x_in: Any, freqs_cis: Any) -> Any:
import torch
with torch.amp.autocast("cuda", enabled = False):
x = torch.view_as_complex(x_in.float().reshape(*x_in.shape[:-1], -1, 2))
x_out = torch.view_as_real(x * freqs_cis.unsqueeze(2)).flatten(3)
return x_out.type_as(x_in)
def _rotation_group(linears: list) -> Any:
"""0 when every part is a plain ``nn.Linear``, the shared ConvRot group when every part rotates its input at the
same group, else None (a mix cannot share one input)."""
from torch import nn
if all(type(lin) is nn.Linear for lin in linears):
return 0
try:
from .diffusion_convrot import convrot_linear_class
cls = convrot_linear_class()
except Exception: # noqa: BLE001
return None
if not all(type(lin) is cls for lin in linears):
return None
groups = {getattr(lin, "convrot_groupsize", None) for lin in linears}
if len(groups) != 1:
return None
group = groups.pop()
if not isinstance(group, int) or group <= 0 or linears[0].in_features % group:
return None
return group
def _fuse_linears(linears: list) -> Any:
"""One Linear over the concatenated outputs, or None; torchao tensors are rebuilt from their data/attribute lists.
Parts that all rotate their input at one ConvRot group give a fused ConvRotLinear at that group."""
import torch
from torch import nn
ws = [lin.weight for lin in linears]
group = _rotation_group(linears)
if group is None:
return None
has_bias = [lin.bias is not None for lin in linears]
if any(has_bias) and not all(has_bias):
return None
cls = type(ws[0])
if any(type(w) is not cls for w in ws) or any(
w.dtype != ws[0].dtype or w.device != ws[0].device for w in ws
):
return None
if any(w.shape[1] != ws[0].shape[1] for w in ws):
return None
if cls in (torch.Tensor, nn.Parameter):
fused_w = torch.cat([w.detach() for w in ws], dim = 0)
else:
names = list(getattr(cls, "tensor_data_names", ()))
attrs = list(getattr(cls, "tensor_attribute_names", ()))
opt_t = list(getattr(cls, "optional_tensor_data_names", ()))
opt_a = list(getattr(cls, "optional_tensor_attribute_names", ()))
if not names:
return None
kwargs: dict = {}
for name in names + opt_t:
parts = [getattr(w, name, None) for w in ws]
if all(p is None for p in parts):
continue
if any(
p is None or p.dim() == 0 or p.shape[0] != w.shape[0] for p, w in zip(parts, ws)
):
return None
kwargs[name] = torch.cat(parts, dim = 0)
for name in attrs + opt_a:
first = getattr(ws[0], name, None)
if any(getattr(w, name, None) != first for w in ws[1:]):
return None
kwargs[name] = first
try:
fused_w = cls(**kwargs)
except Exception: # noqa: BLE001 - an unknown constructor keeps the stock projections
return None
out = nn.Linear(ws[0].shape[1], sum(w.shape[0] for w in ws), bias = False, device = "meta")
out.weight = nn.Parameter(fused_w, requires_grad = False)
if all(has_bias):
out.bias = nn.Parameter(
torch.cat([lin.bias.detach() for lin in linears]), requires_grad = False
)
if group:
from .diffusion_convrot import _install_rotation
_install_rotation(out, group)
return out
def _share_storage(fused: Any, linears: list) -> bool:
from torch import nn
start = 0
views = []
try:
whole = fused.weight.detach()
for lin in linears:
n = lin.weight.shape[0]
view = whole[start : start + n]
if tuple(view.shape) != tuple(lin.weight.shape) or type(view) is not type(
lin.weight.detach()
):
return False
views.append(view)
start += n
for lin, view in zip(linears, views):
lin.weight = nn.Parameter(view, requires_grad = False)
except Exception: # noqa: BLE001
return False
return True
def install(
transformer: Any,
logger: Any = None,
offload_active: bool = False,
) -> dict:
"""Before the first compile; probe + QKV fusion wait for the first forward if the weights are not on the GPU."""
if zimage_fused_disabled():
uninstall() # a previous load may have patched the process-global class
return {"real_rope": False, "fused_qkv": 0}
if type(transformer).__name__ != "ZImageTransformer2DModel":
return {"real_rope": False, "fused_qkv": 0}
if not _patch_class(logger):
return {"real_rope": False, "fused_qkv": 0}
from .diffusion_int8_fused import resident_cuda_cards, run_on_first_call
if resident_cuda_cards(transformer):
return install_modules(transformer, logger, fuse_qkv = not offload_active)
run_on_first_call(
transformer,
"zimage_fused",
lambda t: install_modules(t, logger, fuse_qkv = not offload_active),
)
return {"real_rope": None, "fused_qkv": None}
def _patch_class(logger: Any = None) -> bool:
try:
import importlib
cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor
except Exception: # noqa: BLE001
return False
with _LOCK:
current = cls.__dict__.get("__call__")
if getattr(current, "__unsloth_zimage_fused__", False):
return True
digest = _stock_digest(cls)
if digest not in _FINGERPRINTS["ZSingleStreamAttnProcessor.__call__"]:
if logger is not None:
logger.info(
"diffusion.zimage_fused: stock attention kept: processor differs (%s)", digest
)
return False
_STATE["stock"] = current
cls.__call__ = _make_call(current)
return True
def install_modules(
root: Any,
logger: Any = None,
fuse_qkv: bool = True,
) -> dict:
result = {"real_rope": False, "fused_qkv": 0}
if zimage_fused_disabled() or not _patch_class(logger):
return result
import importlib
import torch
from .diffusion_int8_fused import resident_cuda_cards
cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor
cards = resident_cuda_cards(
root
) # one card, or each card of a layer split (the processor reads its query's card)
if not cards:
return result
with _LOCK:
for index in cards:
if index in _FUSION:
continue
try:
from .diffusion_qwenimage21_rope import inductor_addcmul_is_fma, probe_fusion
_FUSION[index] = (
probe_fusion(torch.device("cuda", index)) if inductor_addcmul_is_fma() else None
)
except Exception: # noqa: BLE001
_FUSION[index] = None
result["real_rope"] = all(_FUSION[index] is not None for index in cards)
for _name, module in root.named_modules():
if not fuse_qkv or type(getattr(module, "processor", None)) is not cls:
continue
if _QKV_ATTR in module.__dict__:
result["fused_qkv"] += 1
continue
parts = [getattr(module, n, None) for n in ("to_q", "to_k", "to_v")]
if any(p is None for p in parts):
continue
fused = _fuse_linears(parts)
if fused is None or not _share_storage(fused, parts):
continue
module.__dict__[_QKV_ATTR] = (fused, *parts)
result["fused_qkv"] += 1
if logger is not None:
logger.info(
"diffusion.zimage_fused: real-arithmetic RoPE %s, fused QKV on %d attention module(s)",
"on" if result["real_rope"] else "off (card's complex multiply not a known fma form)",
result["fused_qkv"],
)
return result
def probe_split_cards(cards: Any) -> tuple:
"""Probe the split's other cards (an unprobed card takes the complex RoPE the regional compile cannot lower).
Returns the cards whose probe passed."""
if zimage_fused_disabled() or not _FUSION:
return ()
import torch
with _LOCK:
for index in cards:
index = int(index)
if index in _FUSION:
continue
try:
from .diffusion_qwenimage21_rope import inductor_addcmul_is_fma, probe_fusion
_FUSION[index] = (
probe_fusion(torch.device("cuda", index)) if inductor_addcmul_is_fma() else None
)
except Exception: # noqa: BLE001 - that card keeps the complex form
_FUSION[index] = None
return tuple(index for index in cards if _FUSION.get(int(index)) is not None)
def uninstall(transformer: Any = None) -> None:
"""Restore the stock processor; drop the fused Linears (the per-row views stay valid) and pending install."""
if transformer is not None:
from .diffusion_int8_fused import cancel_first_call
cancel_first_call(transformer, "zimage_fused")
with _LOCK:
if "stock" in _STATE:
try:
import importlib
cls = importlib.import_module(_MODULE).ZSingleStreamAttnProcessor
if getattr(cls.__dict__.get("__call__"), "__unsloth_zimage_fused__", False):
cls.__call__ = _STATE.pop("stock")
except Exception: # noqa: BLE001
pass
if transformer is not None:
for module in transformer.modules():
module.__dict__.pop(_QKV_ATTR, None)