Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

3.1 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Supplies ``_repeated_blocks`` (on the instance) for verified Diffusers transformers that declare none."""

from __future__ import annotations

from typing import Any, Optional

_VERIFIED_BLOCKS: dict[str, tuple[str, ...]] = {
    "Lumina2Transformer2DModel": ("Lumina2TransformerBlock",),
    "CosmosTransformer3DModel": ("CosmosTransformerBlock",),
    "HiDreamImageTransformer2DModel": (
        "HiDreamImageTransformerBlock",
        "HiDreamImageSingleTransformerBlock",
    ),
}


def discover_repeated_blocks(model: Any) -> tuple[str, ...]:
    names = tuple(getattr(model, "_no_split_modules", None) or ())
    if not names:
        return ()
    try:
        import torch.nn as nn
        counts: dict[str, int] = {}
        for sub in model.modules():
            if isinstance(sub, nn.ModuleList):
                for child in sub.modules():
                    name = type(child).__name__
                    if name in names:
                        counts[name] = counts.get(name, 0) + 1
    except Exception:  # noqa: BLE001 - a probe, never a failed load
        return ()
    return tuple(n for n in names if counts.get(n, 0) >= 2)


def verified_repeated_blocks(class_name: Optional[str]) -> tuple[str, ...]:
    return _VERIFIED_BLOCKS.get(class_name or "", ())


def ensure_repeated_blocks(model: Any) -> tuple[str, ...]:
    blocks = tuple(getattr(model, "_repeated_blocks", None) or ())
    if not blocks:
        verified = verified_repeated_blocks(type(model).__name__)
        found = discover_repeated_blocks(model) if verified else ()
        blocks = tuple(n for n in verified if n in found)
        if not blocks:
            return ()
        model._repeated_blocks = list(blocks)
    # Also when Diffusers declares the blocks itself: its released moe_infer still host-syncs.
    if type(model).__name__ == "HiDreamImageTransformer2DModel":
        install_traceable_moe(model)
    return blocks


def _moe_infer_dense(self: Any, x: Any, flat_expert_indices: Any, flat_expert_weights: Any) -> Any:
    # `where`, not a zero weight: an unpicked expert that overflows in fp16 must add 0, not 0 * inf = NaN.
    import torch

    k = self.num_activated_experts
    idx = flat_expert_indices.view(-1, k)
    weights = flat_expert_weights.view(-1, k).to(x.dtype)
    zero = x.new_zeros(())
    out = torch.zeros_like(x)
    for i, expert in enumerate(self.experts):
        hit = idx == i
        weight = (weights * hit).sum(-1, keepdim = True)
        out = out + torch.where(hit.any(-1, keepdim = True), expert(x) * weight, zero)
    return out


def install_traceable_moe(model: Any) -> int:
    """Swap HiDream's ``moe_infer`` (host ``bincount().cpu()`` breaks every block's graph) for dense experts."""
    patched = 0
    for sub in model.modules():
        if type(sub).__name__ != "MOEFeedForwardSwiGLU":
            continue
        if not callable(getattr(sub, "moe_infer", None)) or not hasattr(sub, "experts"):
            continue
        sub.moe_infer = _moe_infer_dense.__get__(sub)
        patched += 1
    return patched