Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

7.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Reversible, failure-safe, compile-safe accuracy-neutral patches of shared diffusers classes;
uninstall on off-tier loads and on unload; the chat/diffusion arbiter ensures one active pipe, so
class-level state is safe"""

from __future__ import annotations

import logging
import os
from typing import Callable, Optional

import torch
import torch.nn.functional as F

from .diffusion_patch_backend import apply_patch, revert_patch

logger = logging.getLogger(__name__)

# Kill-switch: UNSLOTH_DIFFUSION_EAGER_PATCHES=0 disables the patches (default on).
_ENV_ENABLE = "UNSLOTH_DIFFUSION_EAGER_PATCHES"


def _patches_enabled() -> bool:
    return (os.environ.get(_ENV_ENABLE) or "").strip().lower() not in ("0", "off", "false", "no")


try:
    from diffusers.models.normalization import (
        AdaLayerNormContinuous as _AdaLayerNormContinuous,
        AdaLayerNormZero as _AdaLayerNormZero,
        AdaLayerNormZeroSingle as _AdaLayerNormZeroSingle,
        RMSNorm as _RMSNorm,
    )
except Exception:  # noqa: BLE001
    _AdaLayerNormContinuous = _AdaLayerNormZero = _AdaLayerNormZeroSingle = _RMSNorm = None

try:
    from diffusers.utils.import_utils import is_torch_npu_available as _is_npu
    _NPU = bool(_is_npu())
except Exception:  # noqa: BLE001
    _NPU = False


# Patched forwards, each mirroring diffusers semantics with the fused fast path (``addcmul(input, t1, t2) == input + t1
# * t2``).
def _adaln_continuous_forward(self, x, conditioning_embedding):
    emb = self.linear(self.silu(conditioning_embedding).to(x.dtype))
    scale, shift = torch.chunk(emb, 2, dim = 1)
    # original: self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :]
    return torch.addcmul(shift[:, None, :], self.norm(x), 1 + scale[:, None, :])


def _adaln_zero_forward(
    self,
    x,
    timestep = None,
    class_labels = None,
    hidden_dtype = None,
    emb = None,
):
    if self.emb is not None:
        emb = self.emb(timestep, class_labels, hidden_dtype = hidden_dtype)
    emb = self.linear(self.silu(emb))
    shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, dim = 1)
    # original: self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None]
    x = torch.addcmul(shift_msa[:, None], self.norm(x), 1 + scale_msa[:, None])
    return x, gate_msa, shift_mlp, scale_mlp, gate_mlp


def _adaln_zero_single_forward(
    self,
    x,
    emb = None,
):
    emb = self.linear(self.silu(emb))
    shift_msa, scale_msa, gate_msa = emb.chunk(3, dim = 1)
    x = torch.addcmul(shift_msa[:, None], self.norm(x), 1 + scale_msa[:, None])
    return x, gate_msa


# Filled at install with the ORIGINAL RMSNorm.forward, for the fast path's fallback.
_orig_rmsnorm_forward: Optional[Callable] = None


def _rmsnorm_forward(self, hidden_states):
    # Fall back where F.rms_norm differs from diffusers: NPU, bias, tuple dim, dtype mismatch.
    if _NPU or self.bias is not None or _orig_rmsnorm_forward is None or len(tuple(self.dim)) != 1:
        return _orig_rmsnorm_forward(self, hidden_states)  # type: ignore[misc]
    weight = self.weight
    if weight is None:
        return F.rms_norm(hidden_states, self.dim, None, self.eps)
    if weight.dtype in (torch.float16, torch.bfloat16) and hidden_states.dtype == weight.dtype:
        # Common DiT path (bf16 acts + bf16 weight). Not bit-identical to diffusers: F.rms_norm rounds to bf16 once
        # (diffusers rounds twice), so it is closer to fp64 per call; ComfyUI's comfy/rmsnorm.py uses the same op.
        return F.rms_norm(hidden_states, self.dim, weight, self.eps)
    return _orig_rmsnorm_forward(self, hidden_states)  # type: ignore[misc]


# True for families needing multiple denoisers no single file carries (Ideogram 4), so only a full pipeline load is valid
def _specs():
    return [
        (_AdaLayerNormContinuous, _adaln_continuous_forward),
        (_AdaLayerNormZero, _adaln_zero_forward),
        (_AdaLayerNormZeroSingle, _adaln_zero_single_forward),
        (_RMSNorm, _rmsnorm_forward),
    ]


# Classes whose `forward` we successfully patched, so uninstall reverts exactly those.
_patched: list[type] = []


def install_compile_safe_patches() -> int:
    """Returns the number of patches applied; a second call while installed is a no-op."""
    global _orig_rmsnorm_forward
    if not _patches_enabled():
        uninstall_patches()
        return 0
    if _fp16_rmsnorm_only:
        # full set must install from the stock classes
        uninstall_fp16_rmsnorm()
    if _patched:
        return len(_patched)
    for cls, new_fn in _specs():
        if cls is None:
            continue
        # torch < 2.4 has no F.rms_norm: leave the original rather than install an AttributeError.
        if cls is _RMSNorm and not hasattr(F, "rms_norm"):
            logger.info("eager-patch: skipping RMSNorm (this torch has no F.rms_norm)")
            continue
        if cls is _RMSNorm:
            _orig_rmsnorm_forward = cls.forward
        if apply_patch(cls, "forward", new_fn, match_level = "relaxed"):
            _patched.append(cls)
        else:
            logger.warning(
                "eager-patch: skipping %s (signature mismatch / unavailable)",
                getattr(cls, "__name__", cls),
            )
            if cls is _RMSNorm:
                _orig_rmsnorm_forward = None
    logger.info(
        "eager-patch: installed %d/%d shared diffusion patches", len(_patched), len(_specs())
    )
    return len(_patched)


def uninstall_patches() -> None:
    """Restore every patched class to its exact original ``forward`` (idempotent)."""
    global _orig_rmsnorm_forward
    for cls in list(_patched) + list(_fp16_rmsnorm_only):
        revert_patch(cls, "forward")
    _patched.clear()
    _fp16_rmsnorm_only.clear()
    _orig_rmsnorm_forward = None


# fp16 ``off`` tier: fused RMSNorm alone (Colab T4 Z-Image: 4.4 s of a 29.9 s denoise in decomposed RMSNorm).
# F.rms_norm rounds once (diffusers twice): nearer fp64, not bit-identical.
FP16_RMSNORM_ENV = "UNSLOTH_DIFFUSION_FP16_RMSNORM"
FP16_RMSNORM_CLASSES = ("ZImageTransformer2DModel",)
_fp16_rmsnorm_only: list[type] = []


def fp16_rmsnorm_enabled() -> bool:
    return (os.environ.get(FP16_RMSNORM_ENV) or "").strip().lower() not in (
        "0",
        "off",
        "false",
        "no",
    )


def fp16_rmsnorm_wanted(denoiser: object, dtype: object, speed_mode: Optional[str]) -> bool:
    return (
        fp16_rmsnorm_enabled()
        and _patches_enabled()
        and str(speed_mode or "").strip().lower() == "off"
        and dtype == torch.float16
        and type(denoiser).__name__ in FP16_RMSNORM_CLASSES
    )


def install_fp16_rmsnorm() -> bool:
    """False when the full set already holds it or the patch is refused."""
    global _orig_rmsnorm_forward
    if _fp16_rmsnorm_only:
        return True
    if _patched or _RMSNorm is None or not hasattr(F, "rms_norm"):
        return False
    _orig_rmsnorm_forward = _RMSNorm.forward
    if not apply_patch(_RMSNorm, "forward", _rmsnorm_forward, match_level = "relaxed"):
        _orig_rmsnorm_forward = None
        return False
    _fp16_rmsnorm_only.append(_RMSNorm)
    logger.info(
        "eager-patch: fused RMSNorm on the fp16 eager tier (%s)", ", ".join(FP16_RMSNORM_CLASSES)
    )
    return True


def uninstall_fp16_rmsnorm() -> None:
    global _orig_rmsnorm_forward
    for cls in list(_fp16_rmsnorm_only):
        revert_patch(cls, "forward")
    if _fp16_rmsnorm_only and not _patched:
        _orig_rmsnorm_forward = None
    _fp16_rmsnorm_only.clear()


def fp16_rmsnorm_installed() -> bool:
    return bool(_fp16_rmsnorm_only)


def is_installed() -> bool:
    """True if any compile-safe patch is currently installed."""
    return bool(_patched)