Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

18.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Ideogram 4 assembled per component for transformers 4.x; -fp8 DiTs need fp8 dequant and qkv
split, gated on weight_scale (-nf4 keeps stock path); create_causal_mask kwarg patch self-
disables where supported."""

from __future__ import annotations

import inspect
import json
from pathlib import Path
from typing import Any, Callable, Optional

from loggers import get_logger

from .diffusion_krea2 import load_krea2_text_encoder, load_krea2_tokenizer
from .diffusion_transformer_quant import mark_source_precision

logger = get_logger(__name__)

_CAUSAL_MASK_PATCHED = False


def _patch_create_causal_mask() -> None:
    """Adapt the diffusers Ideogram4 pipeline's ``create_causal_mask`` call to the
    installed transformers signature (see module doc). Idempotent and self-disabling.
    """
    global _CAUSAL_MASK_PATCHED
    if _CAUSAL_MASK_PATCHED:
        return
    import torch
    from diffusers.pipelines.ideogram4 import pipeline_ideogram4 as pipe_mod

    original = pipe_mod.create_causal_mask
    params = inspect.signature(original).parameters

    def create_causal_mask_compat(*args, **kwargs):
        # The pipeline calls this by keyword. Rename inputs_embeds -> input_embeds for the 5.x spelling.
        if "inputs_embeds" in kwargs and "inputs_embeds" not in params and "input_embeds" in params:
            kwargs["input_embeds"] = kwargs.pop("inputs_embeds")
        # Supply cache_position when required and omitted: past_key_values is None, so positions run 0..seq_len-1.
        if "cache_position" in params and "cache_position" not in kwargs:
            embeds = kwargs.get("input_embeds", kwargs.get("inputs_embeds"))
            if embeds is not None:
                kwargs["cache_position"] = torch.arange(embeds.shape[1], device = embeds.device)
        return original(*args, **kwargs)

    pipe_mod.create_causal_mask = create_causal_mask_compat
    _CAUSAL_MASK_PATCHED = True


# The fp8 attention is a fused ``qkv`` matrix (Q/K/V stacked, each ``hidden_size`` rows), read from config so a future
# change cannot mis-split it.
_QKV_SPLIT = ("to_q", "to_k", "to_v")


def _transformer_shard_paths(
    repo_id: str,
    subfolder: str,
    token: Optional[str],
    check_cancelled: Optional[Callable[[], None]] = None,
) -> list[str]:
    """The local safetensors shard paths for ``repo_id/subfolder``.

    Prefers the sharded index; falls back to the single-file name when the subfolder
    ships one file. Resolves through a local dir when ``repo_id`` is a path, else the
    Hub cache.
    """
    check_cancelled = check_cancelled or (lambda: None)
    check_cancelled()
    from huggingface_hub import hf_hub_download

    local_root = Path(repo_id).expanduser()
    if local_root.is_dir():
        sub = local_root / subfolder
        index = sub / "diffusion_pytorch_model.safetensors.index.json"
        if index.is_file():
            weight_map = json.loads(index.read_text(encoding = "utf-8"))["weight_map"]
            return [str(sub / name) for name in sorted(set(weight_map.values()))]
        single = sub / "diffusion_pytorch_model.safetensors"
        if single.is_file():
            return [str(single)]
        raise FileNotFoundError(f"no transformer safetensors under {sub}")

    index_name = f"{subfolder}/diffusion_pytorch_model.safetensors.index.json"
    try:
        index_path = hf_hub_download(repo_id, index_name, token = token)
        weight_map = json.loads(Path(index_path).read_text(encoding = "utf-8"))["weight_map"]
        shards = sorted(set(weight_map.values()))
    except Exception:  # noqa: BLE001 -- single-file subfolder has no index
        shards = ["diffusion_pytorch_model.safetensors"]
    paths = []
    for name in shards:
        check_cancelled()
        paths.append(hf_hub_download(repo_id, f"{subfolder}/{name}", token = token))
    return paths


def _read_transformer_config(repo_id: str, subfolder: str, token: Optional[str]) -> dict[str, Any]:
    """``subfolder/config.json`` as a dict, from a local path or the Hub cache."""
    local = Path(repo_id).expanduser() / subfolder / "config.json"
    if local.is_file():
        return json.loads(local.read_text(encoding = "utf-8"))
    from huggingface_hub import hf_hub_download

    path = hf_hub_download(repo_id, f"{subfolder}/config.json", token = token)
    return json.loads(Path(path).read_text(encoding = "utf-8"))


def _convert_fp8_state_dict(
    raw: dict,
    hidden_size: int,
    dtype,
    check_cancelled: Optional[Callable[[], None]] = None,
) -> dict:
    """Dequantises fp8 weights by their per-channel scale and splits fused qkv into to_q/to_k/to_v."""
    check_cancelled = check_cancelled or (lambda: None)
    import torch

    def dequantize(name: str):
        weight = raw[name].to(torch.float32)
        scale = raw[name + "_scale"].to(torch.float32)
        # Per-channel scale, rank-aware broadcast (correct for a future non-2D quantized tensor).
        return (weight * scale.view(-1, *([1] * (weight.ndim - 1)))).to(dtype)

    converted: dict = {}
    for key, value in raw.items():
        check_cancelled()
        if key.endswith("_scale"):
            continue
        if key + "_scale" not in raw:
            # Dense tensor (norms/biases/embeddings): load as-is.
            converted[key] = value.to(dtype)
            continue
        if key.endswith("attention.qkv.weight"):
            fused = dequantize(key)
            if fused.shape[0] != 3 * hidden_size:
                # equal thirds only holds for full multi-head attention; a GQA export must fail loudly
                raise RuntimeError(
                    f"fused qkv at {key} has {fused.shape[0]} rows, expected "
                    f"{3 * hidden_size}; cannot split into equal Q/K/V blocks"
                )
            base = key[: -len("qkv.weight")]
            for index, proj in enumerate(_QKV_SPLIT):
                block = fused[index * hidden_size : (index + 1) * hidden_size]
                converted[f"{base}{proj}.weight"] = block.clone()
        elif key.endswith("attention.o.weight"):
            converted[key[: -len("o.weight")] + "to_out.0.weight"] = dequantize(key)
        else:
            converted[key] = dequantize(key)
    return converted


def _text_encoder_shard_paths(
    repo_id: str,
    token: Optional[str],
    check_cancelled: Optional[Callable[[], None]] = None,
) -> list[str]:
    """The local safetensors shard paths for ``repo_id/text_encoder`` (index or single file)."""
    check_cancelled = check_cancelled or (lambda: None)
    check_cancelled()
    from huggingface_hub import hf_hub_download

    local_root = Path(repo_id).expanduser()
    if local_root.is_dir():
        sub = local_root / "text_encoder"
        index = sub / "model.safetensors.index.json"
        if index.is_file():
            weight_map = json.loads(index.read_text(encoding = "utf-8"))["weight_map"]
            return [str(sub / name) for name in sorted(set(weight_map.values()))]
        single = sub / "model.safetensors"
        if single.is_file():
            return [str(single)]
        raise FileNotFoundError(f"no text_encoder safetensors under {sub}")

    try:
        index_path = hf_hub_download(
            repo_id, "text_encoder/model.safetensors.index.json", token = token
        )
        weight_map = json.loads(Path(index_path).read_text(encoding = "utf-8"))["weight_map"]
        shards = sorted(set(weight_map.values()))
    except Exception:  # noqa: BLE001 -- single-file text encoder has no index
        shards = ["model.safetensors"]
    paths = []
    for name in shards:
        check_cancelled()
        paths.append(hf_hub_download(repo_id, f"text_encoder/{name}", token = token))
    return paths


def _text_encoder_is_fp8(repo_id: str, token: Optional[str]) -> bool:
    """True when the text_encoder ships the vendor fp8 layout (a ``*.weight_scale`` key)."""
    from huggingface_hub import hf_hub_download

    local_root = Path(repo_id).expanduser()
    if local_root.is_dir():
        index = local_root / "text_encoder" / "model.safetensors.index.json"
        if index.is_file():
            return any(
                k.endswith("_scale")
                for k in json.loads(index.read_text(encoding = "utf-8"))["weight_map"]
            )
    else:
        try:
            index_path = hf_hub_download(
                repo_id, "text_encoder/model.safetensors.index.json", token = token
            )
            weight_map = json.loads(Path(index_path).read_text(encoding = "utf-8"))["weight_map"]
            return any(k.endswith("_scale") for k in weight_map)
        except Exception:  # noqa: BLE001 -- single-file (nf4) text encoder, not fp8
            return False
    import safetensors

    single = local_root / "text_encoder" / "model.safetensors"
    if single.is_file():
        with safetensors.safe_open(str(single), "pt") as handle:
            return any(k.endswith("_scale") for k in handle.keys())
    return False


def load_ideogram4_text_encoder(
    repo_id: str,
    dtype,
    hf_token: Optional[str] = None,
    check_cancelled: Optional[Callable[[], None]] = None,
):
    """The -fp8 encoder needs only the float8 dequant: its keys already match Qwen3-VL, so no qkv rename."""
    check_cancelled = check_cancelled or (lambda: None)
    check_cancelled()
    token = hf_token or None
    is_fp8 = _text_encoder_is_fp8(repo_id, token)
    check_cancelled()
    if not is_fp8:
        return load_krea2_text_encoder(
            repo_id, dtype, hf_token = token, check_cancelled = check_cancelled
        )

    import safetensors
    import torch
    from transformers import AutoConfig, Qwen3VLModel

    from .diffusion_krea2 import remap_rope_parameters

    config_kwargs: dict[str, Any] = {"subfolder": "text_encoder"}
    if token:
        config_kwargs["token"] = token
    config = AutoConfig.from_pretrained(repo_id, **config_kwargs)
    check_cancelled()
    remap_rope_parameters(getattr(config, "text_config", config))

    raw: dict = {}
    for path in _text_encoder_shard_paths(repo_id, token, check_cancelled = check_cancelled):
        check_cancelled()
        with safetensors.safe_open(path, "pt") as handle:
            for key in handle.keys():
                check_cancelled()
                raw[key] = handle.get_tensor(key)

    state_dict: dict = {}
    for key, value in raw.items():
        check_cancelled()
        if key.endswith("_scale"):
            continue
        if key + "_scale" in raw:
            weight = value.to(torch.float32)
            scale = raw[key + "_scale"].to(torch.float32)
            # Rank-aware broadcast (matches _convert_fp8_state_dict).
            state_dict[key] = (weight * scale.view(-1, *([1] * (weight.ndim - 1)))).to(dtype)
        else:
            state_dict[key] = value.to(dtype)

    check_cancelled()
    # build at the target dtype: this ~8B scaffold loads FIRST, so the fp32 default can OOM a 64 GB host
    default_dtype = torch.get_default_dtype()
    torch.set_default_dtype(dtype)
    try:
        model = Qwen3VLModel(config)
        check_cancelled()
        model.to(dtype)
    finally:
        torch.set_default_dtype(default_dtype)
    check_cancelled()
    missing, unexpected = model.load_state_dict(state_dict, strict = False)
    real_missing = [k for k in missing if not k.endswith("inv_freq")]
    if real_missing or unexpected:
        raise RuntimeError(
            f"ideogram4 fp8 text_encoder remap left keys unmatched for {repo_id}: "
            f"missing={real_missing[:8]} unexpected={unexpected[:8]}"
        )
    return model


def ideogram4_repo_is_fp8(repo_id: str, hf_token: Optional[str] = None) -> bool:
    """Checks shard headers for a *.weight_scale key (wider in memory than on disk); any failure returns False."""
    try:
        shard_paths = _transformer_shard_paths(repo_id, "transformer", hf_token or None)
        import safetensors
    except Exception:  # noqa: BLE001 -- treat an unreadable / absent transformer as not fp8
        return False
    for path in shard_paths:
        with safetensors.safe_open(path, "pt") as handle:
            if any(key.endswith("_scale") for key in handle.keys()):
                return True
    return False


def load_ideogram4_transformer(
    repo_id: str,
    subfolder: str,
    dtype,
    hf_token: Optional[str] = None,
    check_cancelled: Optional[Callable[[], None]] = None,
):
    """Vendor fp8 shards are dequantised and renamed; bnb -nf4 repos load via stock from_pretrained."""
    check_cancelled = check_cancelled or (lambda: None)
    check_cancelled()
    import diffusers
    import safetensors
    import torch

    token = hf_token or None
    config = _read_transformer_config(repo_id, subfolder, token)
    check_cancelled()
    shard_paths = _transformer_shard_paths(
        repo_id, subfolder, token, check_cancelled = check_cancelled
    )

    # Check every shard header so a dense-first multi-shard fp8 export still dequantizes.
    is_fp8 = False
    for path in shard_paths:
        check_cancelled()
        with safetensors.safe_open(path, "pt") as handle:
            if any(key.endswith("_scale") for key in handle.keys()):
                is_fp8 = True
                break
    check_cancelled()
    if not is_fp8:
        # Already the diffusers split layout (-nf4): let from_pretrained re-apply its quantization_config.
        model_kwargs: dict[str, Any] = {"subfolder": subfolder, "torch_dtype": dtype}
        if token:
            model_kwargs["token"] = token
        return diffusers.Ideogram4Transformer2DModel.from_pretrained(repo_id, **model_kwargs)

    raw: dict = {}
    for path in shard_paths:
        check_cancelled()
        with safetensors.safe_open(path, "pt") as handle:
            for key in handle.keys():
                check_cancelled()
                raw[key] = handle.get_tensor(key)

    check_cancelled()
    config.pop("quantization_config", None)
    hidden_size = int(config["attention_head_dim"]) * int(config["num_attention_heads"])
    # build at the target dtype: from_config materializes the full ~9B module while the first DiT and the encoder are
    # resident
    default_dtype = torch.get_default_dtype()
    torch.set_default_dtype(dtype)
    try:
        model = diffusers.Ideogram4Transformer2DModel.from_config(config)
    finally:
        torch.set_default_dtype(default_dtype)
    check_cancelled()
    state_dict = _convert_fp8_state_dict(raw, hidden_size, dtype, check_cancelled = check_cancelled)
    check_cancelled()
    missing, unexpected = model.load_state_dict(state_dict, strict = False)
    # inv_freq is built in __init__; any other missing or leftover key must fail loudly.
    real_missing = [k for k in missing if not k.endswith("rotary_emb.inv_freq")]
    if real_missing or unexpected:
        raise RuntimeError(
            f"ideogram4 fp8 remap left keys unmatched for {repo_id}/{subfolder}: "
            f"missing={real_missing[:8]} unexpected={unexpected[:8]}"
        )
    check_cancelled()
    model.to(dtype)
    # Preserve the published source precision after widening the tensors to bf16.
    return mark_source_precision(model, "fp8")


def _hidden_states_on_mask_device(original: Callable[..., Any]) -> Callable[..., Any]:
    """Return the tapped states on ``attention_mask``'s device: under group offload ``text_encoder.device`` reads CPU
    (so the mask lands there) while the states come back on the GPU, and ``encode_prompt``'s multiply raised."""

    def get_text_encoder_hidden_states(text_encoder, token_ids, attention_mask, pos_2d):
        states = original(text_encoder, token_ids, attention_mask, pos_2d)
        device = getattr(attention_mask, "device", None)
        if device is None:
            return states
        return [s.to(device) if getattr(s, "device", device) != device else s for s in states]

    get_text_encoder_hidden_states._unsloth_mask_device = True  # type: ignore[attr-defined]
    return get_text_encoder_hidden_states


def install_text_encoder_device_guard(pipe: Any) -> bool:
    """Shadow the pipeline's static ``_get_text_encoder_hidden_states`` on this instance. Idempotent."""
    original = getattr(pipe, "_get_text_encoder_hidden_states", None)
    if not callable(original):
        return False
    if getattr(original, "_unsloth_mask_device", False):
        return True
    try:
        pipe._get_text_encoder_hidden_states = _hidden_states_on_mask_device(original)
    except Exception:  # noqa: BLE001 - a frozen / slotted pipeline keeps the stock path
        return False
    return True


def load_ideogram4_pipeline(
    repo_id: str,
    dtype,
    hf_token: Optional[str] = None,
    check_cancelled: Optional[Callable[[], None]] = None,
    text_encoder: Any = None,
):
    """Assemble Ideogram4Pipeline from ``repo_id`` per-component (see module doc).

    ``text_encoder`` is a ready encoder (the hosted pre-cast fp8 one); the dense vendor-fp8 encoder is read and
    dequantized only when it is None, so a pre-cast load never touches the ~9 GB of text_encoder shards."""
    check_cancelled = check_cancelled or (lambda: None)
    check_cancelled()
    import diffusers

    # The pipeline's text-encoder call uses a 5.x create_causal_mask signature; adapt it first.
    _patch_create_causal_mask()
    check_cancelled()

    token = hf_token or None
    model_kwargs: dict[str, Any] = {"torch_dtype": dtype}
    if token:
        model_kwargs["token"] = token

    if text_encoder is None:
        text_encoder = load_ideogram4_text_encoder(
            repo_id, dtype, hf_token = token, check_cancelled = check_cancelled
        )
    check_cancelled()
    tokenizer = load_krea2_tokenizer(repo_id, hf_token = token, check_cancelled = check_cancelled)
    check_cancelled()
    transformer = load_ideogram4_transformer(
        repo_id, "transformer", dtype, hf_token = token, check_cancelled = check_cancelled
    )
    check_cancelled()
    # A local checkpoint dir without the file must fail clearly here, else hf_hub_download dies with an opaque HFValidationError
    unconditional_transformer = load_ideogram4_transformer(
        repo_id, "unconditional_transformer", dtype, hf_token = token, check_cancelled = check_cancelled
    )
    check_cancelled()
    vae = diffusers.AutoencoderKLFlux2.from_pretrained(repo_id, subfolder = "vae", **model_kwargs)
    check_cancelled()
    scheduler = diffusers.FlowMatchEulerDiscreteScheduler.from_pretrained(
        repo_id, subfolder = "scheduler", token = token
    )
    check_cancelled()
    logger.info("diffusion.ideogram4: assembled pipeline from %s per-component", repo_id)
    pipe = diffusers.Ideogram4Pipeline(
        scheduler = scheduler,
        vae = vae,
        text_encoder = text_encoder,
        tokenizer = tokenizer,
        transformer = transformer,
        unconditional_transformer = unconditional_transformer,
    )
    install_text_encoder_device_guard(pipe)
    return pipe