Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

49.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
#
# The transformer, scheduler and pipeline below are a port of OpenMOSS-Team's ``pipeline_moss_image.py``
# (Apache-2.0, https://huggingface.co/OpenMOSS-Team/MOSS-Image-SFT), keeping its state-dict names and numerics.

"""MOSS-Image in-tree diffusers pipeline, no remote code; numerics bit-identical to reference, only
default sigmas differ"""

from __future__ import annotations

import math
import os
from typing import Any, Callable, Optional

import torch
from torch import nn
from torch.nn import functional as F

from loggers import get_logger

logger = get_logger(__name__)

MOSS_IMAGE_FAMILY_NAME = "moss-image"
MOSS_IMAGE_PIPELINE_CLASS = "MOSSImagePipeline"
MOSS_IMAGE_TRANSFORMER_CLASS = "MOSSImageTransformer"
MOSS_IMAGE_SCHEDULER_CLASS = "MOSSImageScheduler"
# The Qwen3.5 text encoder class (``transformers.Qwen3_5Model``) first shipped in transformers 5.3.
MIN_TRANSFORMERS = "5.3.0"
SIGMAS_ENV = "UNSLOTH_MOSS_SIGMAS"
SIGMA_SCHEDULES = ("sgm_uniform", "training_grid")
# Set by diffusion_moss_fused, read only inside a compiled block.
FUSED_QKV_ATTR = "_unsloth_moss_fused_qkv"
LINEAR_RUNNER_ATTR = "_unsloth_moss_linear"
QK_ROPE_ATTR = "_unsloth_moss_qk_rope"
# Kill switch for the per-render text-refiner cache (``refine_text``): 0 refines on every step like the reference.
TEXT_CACHE_ENV = "UNSLOTH_MOSS_TEXT_CACHE"
DEFAULT_UNCOND_SKIP_LAYERS = (22,)
# SFT skips nothing; Base skips block 22 for the first half of the sigma schedule. Env 0 = block 22 every step.
REFERENCE_SKIP_ENV = "UNSLOTH_MOSS_REFERENCE_SKIP"
BASE_UNCOND_SKIP_END = 0.5
_UNSET = object()


def _env_off(name: str) -> bool:
    return (os.environ.get(name) or "").strip().lower() in ("0", "off", "false", "no")


def _compute_dtype(linear: nn.Module) -> torch.dtype:
    """The dtype a Linear's input is cast to: the reference casts to ``weight.dtype``, which is the storage dtype under
    diffusers' ``GGUFLinear`` (uint8), so a ``compute_dtype`` the layer declares wins. Same dtype for a plain Linear."""
    declared = getattr(linear, "compute_dtype", None)
    return declared if isinstance(declared, torch.dtype) else linear.weight.dtype


class _RMSNorm(nn.Module):
    def __init__(self, dim):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(dim))

    def forward(self, x):
        y = x.float() * torch.rsqrt(x.float().square().mean(-1, keepdim = True) + 1e-6)
        return y.to(x.dtype) * self.weight


def _situ(hidden, gate):
    return (4.0 * torch.tanh(gate / 4.0) * torch.sigmoid(gate)) * (25.0 * torch.tanh(hidden / 25.0))


def _validate_skip_layers(
    layers,
    depth,
    name = "skip_layers",
):
    layers = () if layers is None else tuple(layers)
    if any(type(index) is not int or not 0 <= index < depth for index in layers):
        raise ValueError(f"{name} must contain integer block indices in [0, {depth}).")
    return frozenset(layers)


def _length_dynamic(*tensors) -> None:
    """Hint dim 1 (the text length) of the refiner block inputs as dynamic. The pipeline refines the text once per render
    OUTSIDE ``forward`` (``refine_text``), where Studio's per-forward prompt-length arming does not reach, so the
    compiled text block otherwise specialised on the first length and recompiled on the uncond one. A no-op eagerly."""
    try:
        if torch.compiler.is_compiling():
            return
        from torch._dynamo import maybe_mark_dynamic
    except Exception:  # noqa: BLE001 - no dynamo, nothing to hint
        return
    for tensor in tensors:
        if isinstance(tensor, torch.Tensor) and tensor.dim() > 1:
            maybe_mark_dynamic(tensor, 1)


def _rope_tables(rope):
    """``(cos, sin)`` duplicated to the full head dim, shaped to broadcast over heads: the split-half layout the
    reference's ``_rotate`` builds on every call, built once."""
    sin, cos = rope.chunk(2, -1)
    sin = torch.cat((sin, sin), -1).unsqueeze(-2)
    cos = torch.cat((cos, cos), -1).unsqueeze(-2)
    return cos, sin


def _rotate(x, tables):
    cos, sin = tables
    a, b = x.chunk(2, -1)
    return x * cos + torch.cat((-b, a), -1) * sin


class _AttnProcessor:
    """Holds the attention backend diffusers' ``set_attention_backend`` assigns (None = the dispatcher's active one)."""

    _attention_backend = None
    _parallel_config = None


def _dispatch_attention():
    try:
        from diffusers.models.attention_dispatch import dispatch_attention_fn
    except Exception:  # noqa: BLE001 - an older diffusers without the dispatcher
        return None
    return dispatch_attention_fn


_DISPATCH = _dispatch_attention()


def _attention(
    q,
    k,
    v,
    tables,
    heads,
    backend = None,
):
    shape = (*q.shape[:2], heads, q.shape[-1] // heads)
    q = _rotate(q.reshape(shape), tables)
    k = _rotate(k.reshape(shape), tables)
    v = v.reshape(shape)
    if _DISPATCH is None:
        out = F.scaled_dot_product_attention(
            q.to(v.dtype).transpose(1, 2), k.to(v.dtype).transpose(1, 2), v.transpose(1, 2)
        ).transpose(1, 2)
    else:
        out = _DISPATCH(q.to(v.dtype), k.to(v.dtype), v, backend = backend)
    return out.flatten(2)


def _attention_heads_first(
    q,
    k,
    v,
    backend = None,
):
    """Attention over already normalised and rotated [B, H, S, D] q / k / v (the dispatcher takes [B, S, H, D] views)."""
    q, k, v = q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
    if _DISPATCH is None:
        out = F.scaled_dot_product_attention(
            q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
        ).transpose(1, 2)
    else:
        out = _DISPATCH(q, k, v, backend = backend)
    return out.flatten(2)


try:  # the mixin is what ``ModelMixin.set_attention_backend`` looks for
    from diffusers.models.attention import AttentionModuleMixin as _AttentionMixin
except Exception:  # noqa: BLE001 - older diffusers: attention stays on SDPA

    class _AttentionMixin:  # type: ignore[no-redef]
        pass


class _TextBlock(nn.Module, _AttentionMixin):
    _supports_qkv_fusion = False

    def __init__(self, dim, ffn_dim, heads):
        super().__init__()
        self.heads = heads
        self.processor = _AttnProcessor()
        self.norm1, self.norm2 = _RMSNorm(dim), _RMSNorm(dim)
        self.q_norm, self.k_norm = _RMSNorm(dim // heads), _RMSNorm(dim // heads)
        self.qkv = nn.Linear(dim, 3 * dim)
        self.proj = nn.Linear(dim, dim)
        self.ffn_gate, self.ffn_up = nn.Linear(dim, ffn_dim), nn.Linear(dim, ffn_dim)
        self.ffn_down = nn.Linear(ffn_dim, dim)

    def forward(self, x, rope):
        q, k, v = self.qkv(self.norm1(x)).chunk(3, -1)
        q = self.q_norm(q.unflatten(-1, (self.heads, -1))).flatten(2)
        k = self.k_norm(k.unflatten(-1, (self.heads, -1))).flatten(2)
        x = x + self.proj(_attention(q, k, v, rope, self.heads, self.processor._attention_backend))
        y = self.norm2(x)
        return x + self.ffn_down(_situ(self.ffn_up(y), self.ffn_gate(y)))


class _TextProjection(nn.Module):
    def __init__(self, text_dim, dim, ffn_dim, heads):
        super().__init__()
        self.proj = nn.Linear(text_dim, dim)
        self.blocks = nn.ModuleList([_TextBlock(dim, ffn_dim, heads) for _ in range(2)])

    def forward(self, x, rope):
        x = self.proj(x.to(_compute_dtype(self.proj)))
        _length_dynamic(x, *rope)
        for block in self.blocks:
            x = block(x, rope)
        return x


class _Block(nn.Module, _AttentionMixin):
    _supports_qkv_fusion = False

    def __init__(self, dim, ffn_dim, heads):
        super().__init__()
        self.heads = heads
        self.processor = _AttnProcessor()
        self.norm1, self.norm2 = _RMSNorm(dim), _RMSNorm(dim)
        self.q_norm, self.k_norm = _RMSNorm(dim // heads), _RMSNorm(dim // heads)
        self.proj = nn.ModuleDict(
            {name: nn.Linear(dim, dim, bias = False) for name in ("q", "k", "v", "o")}
        )
        self.proj["ffn_up_gate"] = nn.Linear(dim, 2 * ffn_dim, bias = False)
        self.proj["ffn_down"] = nn.Linear(ffn_dim, dim, bias = False)

    def _linear(self, name, x):
        run = self.__dict__.get(LINEAR_RUNNER_ATTR) if torch.compiler.is_compiling() else None
        return self.proj[name](x) if run is None else run(self.proj[name], x)

    def _qkv(self, y):
        fused = self.__dict__.get(FUSED_QKV_ATTR) if torch.compiler.is_compiling() else None
        if fused is not None:
            linear, q, k, v, run = fused
            # only while q / k / v are still the modules it was built from (a LoRA wrapper replaces them)
            if self.proj["q"] is q and self.proj["k"] is k and self.proj["v"] is v:
                return (linear(y) if run is None else run(linear, y)).chunk(3, -1)
        return self._linear("q", y), self._linear("k", y), self._linear("v", y)

    def forward(self, x, rope, modulation):
        shift_a, scale_a, gate_a, shift_f, scale_f, gate_f = modulation.unbind(1)
        y = self.norm1(x) * (1 + scale_a[:, None]) + shift_a[:, None]
        dtype = _compute_dtype(self.proj["q"])
        y = y.to(dtype)
        q, k, v = self._qkv(y)
        fused = self.__dict__.get(QK_ROPE_ATTR) if torch.compiler.is_compiling() else None
        if fused is not None and q.dtype == torch.bfloat16:
            q, k, v = fused(
                q, k, v, self.q_norm.weight, self.k_norm.weight, rope[0], rope[1], self.heads
            )
            attn = _attention_heads_first(q, k, v, self.processor._attention_backend)
        else:
            q = self.q_norm(q.unflatten(-1, (self.heads, -1))).flatten(2)
            k = self.k_norm(k.unflatten(-1, (self.heads, -1))).flatten(2)
            attn = _attention(q, k, v, rope, self.heads, self.processor._attention_backend)
        x = x + self._linear("o", attn) * gate_a[:, None]
        y = self.norm2(x) * (1 + scale_f[:, None]) + shift_f[:, None]
        hidden, gate = self._linear("ffn_up_gate", y.to(dtype)).chunk(2, -1)
        return x + self._linear("ffn_down", _situ(hidden, gate)) * gate_f[:, None]


class _RoPE(nn.Module):
    _MAX_CACHED = 8

    def __init__(self, head_dim, base):
        super().__init__()
        self.num_bands = head_dim // 8
        self.base = base
        self.register_buffer("bands", self._default_bands(torch.device("cpu")))
        self._tables: dict = {}

    def _default_bands(self, device):
        # Match training's CPU fp32 construction, including under meta init.
        exponent = torch.arange(self.num_bands, device = "cpu", dtype = torch.float32) / self.num_bands
        return (1 / (self.base**exponent)).to(device)

    def _apply(
        self,
        fn,
        recurse = True,
    ):
        super()._apply(fn, recurse = recurse)
        # .float() cannot recover frequencies already rounded by .to(bf16).
        self.bands = self._default_bands(self.bands.device)
        self._tables = {}
        return self

    def _load_from_state_dict(self, *args, **kwargs):
        super()._load_from_state_dict(*args, **kwargs)
        # Frequencies are derived constants; the key stays for checkpoint compatibility.
        self.bands = self._default_bands(self.bands.device)
        self._tables = {}

    def forward(self, height, width, text_length, device):
        """The reference's phase table ``[sin | cos]`` in fp32, [1, H*W + L, head_dim]."""
        if self.bands.dtype != torch.float32 or self.bands.device.type == "meta":
            self.bands = self._default_bands(device)
        bands = self.bands.to(device)
        h, w = torch.meshgrid(
            torch.arange(height, device = device), torch.arange(width, device = device), indexing = "ij"
        )
        image = (
            torch.stack((torch.zeros_like(h), torch.full_like(h, text_length), h, w), -1)
            .flatten(0, 1)
            .float()
        )
        image[:, 2] -= (height - 1) / 2
        image[:, 3] -= (width - 1) / 2
        text = torch.zeros(text_length, 4, device = device, dtype = torch.float32)
        text[:, 1] = torch.arange(text_length, device = device)
        phase = torch.cat((image, text), 0).unsqueeze(-1) * bands
        return torch.cat((phase.sin(), phase.cos()), -2).flatten(-2).unsqueeze(0)

    def tables(self, height, width, text_length, device, dtype):
        """``((cos, sin) for image + text, (cos, sin) for the text alone)`` in ``dtype``, cached per key."""
        device = torch.device(device)
        key = (int(height), int(width), int(text_length), device.type, device.index, dtype)
        hit = self._tables.get(key)
        if hit is not None:
            return hit
        rope = self.forward(height, width, text_length, device).to(dtype)
        full = _rope_tables(rope)
        text = _rope_tables(rope[:, height * width :])
        if len(self._tables) >= self._MAX_CACHED:
            self._tables.pop(next(iter(self._tables)))
        entry = self._tables[key] = (full, text)
        return entry


def _model_bases():
    from diffusers.configuration_utils import ConfigMixin
    from diffusers.models.modeling_utils import ModelMixin
    return ModelMixin, ConfigMixin


_ModelMixin, _ConfigMixin = _model_bases()
from diffusers.loaders import PeftAdapterMixin as _PeftAdapterMixin  # noqa: E402  (LoRA: load_lora_adapter)
from diffusers.configuration_utils import register_to_config  # noqa: E402


def _float64_device(device):
    """Where the float64 timestep sinusoid is built: MPS and Arc XPUs have no float64, so there it is the CPU."""
    if device.type in ("cuda", "cpu"):
        return device
    if device.type == "xpu" and getattr(torch.xpu.get_device_properties(device), "has_fp64", False):
        return device
    return torch.device("cpu")


class MOSSImageTransformer(_ModelMixin, _ConfigMixin, _PeftAdapterMixin):
    """MOSS-Image DiT: a text refiner, then 42 single-stream joint blocks over [image ; text] sharing one AdaLN.

    Each sample runs at its true text length, text refinement included, so padded tokens never reach attention."""

    _repeated_blocks = ["_Block", "_TextBlock"]
    _no_split_modules = ["_Block", "_TextBlock"]
    _skip_layerwise_casting_patterns = ["rope", "norm"]

    @register_to_config
    def __init__(
        self,
        dim = 3072,
        ffn_dim = 8192,
        num_heads = 24,
        num_layers = 42,
        text_dim = 4096,
        freq_dim = 256,
        latent_channels = 128,
        rope_base = 10000.0,
    ):
        super().__init__()
        if dim % num_heads or (dim // num_heads) % 8 or freq_dim % 2:
            raise ValueError(
                "Require dim divisible by heads, head_dim divisible by 8, and even freq_dim."
            )
        self.video_embedder = nn.Linear(latent_channels, dim)
        self.text_embedder = _TextProjection(text_dim, dim, ffn_dim, num_heads)
        self.rope = _RoPE(dim // num_heads, rope_base)
        self.time_embedding = nn.Sequential(
            nn.Linear(freq_dim, dim), nn.SiLU(), nn.Linear(dim, dim)
        )
        self.shared_block_modulation = nn.Linear(dim, 6 * dim)
        self.layers = nn.ModuleList([_Block(dim, ffn_dim, num_heads) for _ in range(num_layers)])
        self.final_norm_video = _RMSNorm(dim)
        self.final_linear_video = nn.Linear(dim, latent_channels, bias = False)
        self.video_head_modulation = nn.Linear(dim, 2 * dim)

    def load_training_state_dict(
        self,
        state_dict,
        *,
        prefix = "",
    ):
        """Load a consolidated raw/EMA model mapping, strictly; no silent key drops."""
        if prefix:
            if any(not key.startswith(prefix) for key in state_dict):
                raise ValueError("All keys must start with the explicit prefix.")
            state_dict = {key[len(prefix) :]: value for key, value in state_dict.items()}
        return self.load_state_dict(state_dict, strict = True)

    def _activation_dtype(self):
        return _compute_dtype(self.video_embedder)

    def refine_text(self, encoder_hidden_states):
        """The text refiner over [B, L, text_dim] at full length L (no padding): [B, L, dim]. Depends on neither the
        latent nor the timestep, so it is step-invariant; the RoPE text positions depend only on L."""
        length = encoder_hidden_states.shape[1]
        device = encoder_hidden_states.device
        _, text_tables = self.rope.tables(1, 1, length, device, self._activation_dtype())
        return self.text_embedder(encoder_hidden_states, text_tables)

    def _time_modulation(self, timestep, batch, device):
        times = torch.as_tensor(timestep, device = device, dtype = torch.float32).reshape(-1)
        times = times.expand(batch)
        # Training computes the sinusoid in float64 and feeds float32 into its MLP.
        half = self.config.freq_dim // 2
        where = _float64_device(times.device)
        freq = 10000 ** (-torch.arange(half, device = where, dtype = torch.float64) / half)
        # Moved first, then cast: an MPS copy straight into a float64 CPU tensor fails or writes nothing.
        phase = times.to(where).double()[:, None] * freq
        embedding = torch.cat((phase.cos(), phase.sin()), -1).float().to(times.device)
        time = self.time_embedding(embedding.to(_compute_dtype(self.time_embedding[0]))).float()
        mod = self.shared_block_modulation(
            time.to(_compute_dtype(self.shared_block_modulation))
        ).view(batch, 6, -1)
        head = self.video_head_modulation(time.to(_compute_dtype(self.video_head_modulation))).view(
            batch, 2, -1
        )
        return mod, head

    def forward(
        self,
        hidden_states,
        timestep,
        encoder_hidden_states = None,
        encoder_attention_mask = None,
        return_dict = True,
        skip_layers = None,
        refined_text = None,
    ):
        """``refined_text`` ([B, L, dim], from ``refine_text``) replaces ``encoder_hidden_states`` and skips the text
        refiner; give exactly one. A mask applies to ``encoder_hidden_states`` only (refined text is unpadded)."""
        skip_layers = _validate_skip_layers(skip_layers, len(self.layers))
        if hidden_states.ndim != 4 or hidden_states.shape[1] != self.config.latent_channels:
            raise ValueError("hidden_states must be [B, latent_channels, H, W].")
        batch, _, height, width = hidden_states.shape
        if (encoder_hidden_states is None) == (refined_text is None):
            raise ValueError("Give exactly one of encoder_hidden_states and refined_text.")
        text_in = encoder_hidden_states if refined_text is None else refined_text
        if text_in.ndim != 3 or text_in.shape[0] != batch:
            raise ValueError("encoder_hidden_states must be [B, S, text_dim].")
        mask = encoder_attention_mask
        if mask is not None:
            if refined_text is not None:
                raise ValueError(
                    "refined_text is unpadded; pass no encoder_attention_mask with it."
                )
            # The reference checks a mask on every call; it costs a host sync, so only a mask that is given is checked.
            if mask.shape != text_in.shape[:2] or not torch.all((mask == 0) | (mask == 1)):
                raise ValueError("encoder_attention_mask must be a binary [B, S] mask.")
        device = hidden_states.device
        mod, head = self._time_modulation(timestep, batch, device)
        dtype = self._activation_dtype()
        outputs = []
        for index in range(batch):
            if mask is None:
                context = text_in[index : index + 1]
            else:
                context = text_in[index : index + 1, mask[index].bool()]
            length = context.shape[1]
            if length == 0:
                raise ValueError("Each sample needs at least one valid text token.")
            video = hidden_states[index : index + 1].flatten(2).transpose(1, 2)
            video = self.video_embedder(video.to(_compute_dtype(self.video_embedder)).contiguous())
            # The reference casts RoPE to the video activation dtype.
            full_tables, text_tables = self.rope.tables(height, width, length, device, video.dtype)
            text = self.text_embedder(context, text_tables) if refined_text is None else context
            x = torch.cat((video, text.to(video.dtype)), 1)
            sample_mod = mod[index : index + 1]
            for layer_index, layer in enumerate(self.layers):
                if layer_index not in skip_layers:
                    x = layer(x, full_tables, sample_mod)
            x = self.final_norm_video(x[:, : height * width].to(self.final_norm_video.weight.dtype))
            x = x * (1 + head[index : index + 1, 1, None]) + head[index : index + 1, 0, None]
            x = self.final_linear_video(x.to(_compute_dtype(self.final_linear_video)))
            outputs.append(x.transpose(1, 2).reshape(1, -1, height, width))
        sample = outputs[0] if batch == 1 else torch.cat(outputs)
        if not return_dict:
            return (sample,)
        from diffusers.models.modeling_outputs import Transformer2DModelOutput

        return Transformer2DModelOutput(sample = sample)


from diffusers import FlowMatchEulerDiscreteScheduler  # noqa: E402


def _time_snr_shift(shift: float, t):
    if shift == 1.0:
        return t
    return shift * t / (1 + (shift - 1) * t)


def sgm_uniform_sigmas(
    steps: int,
    shift: float,
    num_train_timesteps: int = 1000,
) -> list:
    """ComfyUI ``sgm_uniform`` over ``ModelSamplingDiscreteFlow`` (multiplier 1), in its fp32 ops, without the
    terminal zero: ``linspace(sigma_max, sigma_min, steps + 1)[:-1]`` re-shifted, as ``comfy.samplers.normal_scheduler``
    with ``sgm=True``."""
    grid = _time_snr_shift(shift, torch.arange(1, num_train_timesteps + 1, 1) / num_train_timesteps)
    sigma_min, sigma_max = grid[0], grid[-1]
    timesteps = torch.linspace(sigma_max, sigma_min, steps + 1)[:-1]
    return [float(_time_snr_shift(shift, ts)) for ts in timesteps]


def training_grid_sigmas(
    steps: int,
    shift: float,
    training_shift: float,
    total: int = 1000,
) -> list:
    """The reference pipeline's schedule: shifted linspace snapped to the nearest of the training grid's sigmas."""

    def grid(count, s):
        lower = s / (total + s - 1)
        values = torch.linspace(1, lower, count)
        return s * values / (1 + (s - 1) * values)

    sigmas = grid(steps, shift)
    training = grid(total, training_shift).flip(0)
    upper = torch.searchsorted(training, sigmas).clamp(max = total - 1)
    lower = (upper - 1).clamp(min = 0)
    nearest = torch.where(
        (sigmas - training[lower]).abs() <= (training[upper] - sigmas).abs(), lower, upper
    )
    return torch.unique_consecutive(training[nearest]).tolist()


def default_sigma_schedule() -> str:
    raw = (os.environ.get(SIGMAS_ENV) or "").strip().lower()
    return raw if raw in SIGMA_SCHEDULES else "sgm_uniform"


class MOSSImageScheduler(FlowMatchEulerDiscreteScheduler):
    """Flow-matching Euler at shift 5. ``sigma_schedule``: ``sgm_uniform`` (ComfyUI, the Studio default) or
    ``training_grid`` (the reference pipeline's nearest-training-sigma snapping); None reads ``UNSLOTH_MOSS_SIGMAS``."""

    @register_to_config
    def __init__(
        self,
        num_train_timesteps = 1000,
        shift = 5.0,
        training_shift = 1.0,
        sigma_schedule = None,
    ):
        super().__init__(num_train_timesteps = num_train_timesteps, shift = 1.0)
        self.register_to_config(
            shift = shift, training_shift = training_shift, sigma_schedule = sigma_schedule
        )

    def resolved_sigma_schedule(self) -> str:
        chosen = getattr(self.config, "sigma_schedule", None)
        return chosen if chosen in SIGMA_SCHEDULES else default_sigma_schedule()

    def set_timesteps(
        self,
        num_inference_steps = 50,
        device = None,
        sigmas = None,
        **kwargs,
    ):
        if sigmas is None:
            if num_inference_steps is None or num_inference_steps < 1:
                raise ValueError("Steps and shifts must be positive.")
            if self.config.shift <= 0 or self.config.training_shift <= 0:
                raise ValueError("Steps and shifts must be positive.")
            if self.resolved_sigma_schedule() == "training_grid":
                sigmas = training_grid_sigmas(
                    num_inference_steps,
                    self.config.shift,
                    self.config.training_shift,
                    self.config.num_train_timesteps,
                )
            else:
                sigmas = sgm_uniform_sigmas(
                    num_inference_steps, self.config.shift, self.config.num_train_timesteps
                )
        # Already shifted: no second shift.
        self._shift = 1.0
        super().set_timesteps(device = device, sigmas = list(sigmas))


from diffusers import DiffusionPipeline  # noqa: E402


def is_base_checkpoint(identity: Optional[str]) -> bool:
    folded = "".join(c for c in str(identity or "").lower() if c.isalnum())
    return "mossimagebase" in folded


def reference_uncond_skip(identity: Optional[str]) -> tuple[tuple[int, ...], float]:
    """``(uncond_skip_layers, uncond_skip_end)`` for a checkpoint."""
    if _env_off(REFERENCE_SKIP_ENV):
        return DEFAULT_UNCOND_SKIP_LAYERS, 1.0
    if is_base_checkpoint(identity):
        return DEFAULT_UNCOND_SKIP_LAYERS, BASE_UNCOND_SKIP_END
    return (), 1.0


def percent_to_sigma(percent: float, shift: float) -> float:
    """As ComfyUI ``ModelSamplingDiscreteFlow.percent_to_sigma`` (multiplier 1)."""
    if percent <= 0.0:
        return 1.0
    if percent >= 1.0:
        return 0.0
    t = 1.0 - percent
    return shift * t / (1 + (shift - 1) * t)


def _offloaded(module: Any) -> bool:
    """Accelerate / diffusers offload hooks move weights per call: the pre-refined text path calls a submodule outside
    the forward, so an offloaded transformer keeps the reference per-step refine."""
    if module is None:
        return True
    for sub in module.modules():
        if getattr(sub, "_hf_hook", None) is not None:
            return True
        registry = getattr(sub, "_diffusers_hook", None)
        hooks = getattr(registry, "hooks", None) or {}
        if hooks:
            return True
    return False


def comfy_noise(shape, generator, device):
    """Initial fp32 noise as ComfyUI draws it: ``torch.randn`` on the CPU from the seed (``comfy.sample.prepare_noise``).

    A CPU generator is used as given (the reference pipeline's ``randn_tensor`` draws a CPU generator's noise on the
    CPU too, so the two agree bit for bit); an accelerator generator contributes only its seed, so a Studio seed renders
    what ComfyUI renders for it. A list holds one generator per image; None draws from the global CPU stream."""
    gens = list(generator) if isinstance(generator, (list, tuple)) else [generator]

    def _cpu(g):
        if g is None or g.device.type == "cpu":
            return g
        return torch.Generator("cpu").manual_seed(int(g.initial_seed()))

    if len(gens) == 1:
        noise = torch.randn(shape, generator = _cpu(gens[0]), dtype = torch.float32)
    else:
        if len(gens) != shape[0]:
            raise ValueError(
                f"Got {len(gens)} generators for {shape[0]} images; pass one per image."
            )
        row = (1, *shape[1:])
        noise = torch.cat([torch.randn(row, generator = _cpu(g), dtype = torch.float32) for g in gens])
    return noise.to(device)


class MOSSImagePipeline(DiffusionPipeline):
    model_cpu_offload_seq = "text_encoder->transformer->vae"
    _optional_components = ["text_encoder", "tokenizer", "vae"]
    _callback_tensor_inputs = ["latents"]

    def __init__(
        self,
        transformer,
        scheduler,
        text_encoder = None,
        tokenizer = None,
        vae = None,
    ):
        super().__init__()
        self.register_modules(
            transformer = transformer,
            scheduler = scheduler,
            text_encoder = text_encoder,
            tokenizer = tokenizer,
            vae = vae,
        )
        from diffusers.image_processor import VaeImageProcessor

        self.image_processor = VaeImageProcessor(vae_scale_factor = 16)
        self._interrupt = False

    @property
    def interrupt(self) -> bool:
        return self._interrupt

    # LoRA on the transformer only; diffusion_lora_files folds alpha and splits fused qkv first.
    def load_lora_weights(
        self,
        state_dict,
        adapter_name: Optional[str] = None,
        **_kwargs,
    ):
        metadata = None
        if not isinstance(state_dict, dict):
            from diffusers.utils.state_dict_utils import _load_sft_state_dict_metadata
            from safetensors.torch import load_file

            # lora_alpha / rank of a diffusers-saved adapter live only in the file's metadata.
            metadata = _load_sft_state_dict_metadata(str(state_dict))
            state_dict = load_file(str(state_dict))
        converted = {}
        for key, value in state_dict.items():
            for prefix in ("diffusion_model.", "model.diffusion_model.", "transformer."):
                if key.startswith(prefix):
                    key = key[len(prefix) :]
                    break
            converted["transformer." + key] = value
        self.transformer.load_lora_adapter(
            converted, adapter_name = adapter_name, prefix = "transformer", metadata = metadata
        )

    def set_adapters(
        self,
        adapter_names,
        adapter_weights = None,
    ):
        self.transformer.set_adapters(adapter_names, weights = adapter_weights)

    def get_list_adapters(self) -> dict:
        config = getattr(self.transformer, "peft_config", None) or {}
        return {"transformer": list(config)} if config else {}

    def unload_lora_weights(self):
        if getattr(self.transformer, "peft_config", None):
            self.transformer.unload_lora()

    @torch.no_grad()
    def encode_prompt(
        self,
        prompt,
        max_sequence_length = 4096,
        device = None,
    ):
        """``(embeds [B, L, C], mask [B, L] bool)``: each prompt through the chat template, encoded unpadded on its own
        (Qwen3.5's gated-delta layers see no padding), then right-padded into one batch."""
        if self.text_encoder is None or self.tokenizer is None:
            raise ValueError("Text encoder/tokenizer required for string prompts.")
        prompts = [prompt] if isinstance(prompt, str) else list(prompt)
        if not prompts or not all(isinstance(p, str) for p in prompts):
            raise ValueError("prompt must be a string or a nonempty list of strings.")
        device = device or self._execution_device
        rows = []
        cache: dict = {}
        for value in prompts:
            if value in cache:
                rows.append(cache[value])
                continue
            formatted = self.tokenizer.apply_chat_template(
                [{"role": "user", "content": value}],
                tokenize = False,
                add_generation_prompt = True,
                enable_thinking = True,
            )
            inputs = self.tokenizer(
                formatted, return_tensors = "pt", truncation = True, max_length = max_sequence_length
            ).to(device)
            output = self.text_encoder(**inputs, use_cache = False, return_dict = True)
            cache[value] = output.last_hidden_state[0]
            rows.append(cache[value])
        length = max(row.shape[0] for row in rows)
        embeds = rows[0].new_zeros(len(rows), length, rows[0].shape[-1])
        mask = torch.zeros(len(rows), length, device = embeds.device, dtype = torch.bool)
        for index, row in enumerate(rows):
            embeds[index, : row.shape[0]] = row
            mask[index, : row.shape[0]] = True
        return embeds, mask

    def _branch(self, embeds, mask, refine: bool) -> dict:
        """Transformer kwargs for one CFG branch. Equal-length rows need no mask (no host sync in the forward); with
        ``refine`` the text refiner runs here, once for the whole render."""
        full = mask is None or bool(mask.all())
        kwargs: dict = {}
        if full and refine:
            kwargs["refined_text"] = self.transformer.refine_text(embeds)
        elif full:
            kwargs["encoder_hidden_states"] = embeds
        else:
            kwargs["encoder_hidden_states"] = embeds
            kwargs["encoder_attention_mask"] = mask
        return kwargs

    @torch.no_grad()
    def __call__(
        self,
        prompt = None,
        *,
        height = 1024,
        width = 1024,
        num_inference_steps = 50,
        guidance_scale = 3.0,
        negative_prompt = "",
        num_images_per_prompt = 1,
        generator = None,
        latents = None,
        prompt_embeds = None,
        prompt_attention_mask = None,
        negative_prompt_embeds = None,
        negative_prompt_attention_mask = None,
        max_sequence_length = 4096,
        output_type = "pil",
        return_dict = True,
        uncond_skip_layers = _UNSET,
        uncond_skip_start = 0.0,
        uncond_skip_end = None,
        slg_layers = None,
        slg_scale = 0.0,
        slg_start = 0.0,
        slg_stop = 1.0,
        sigmas = None,
        callback_on_step_end: Optional[Callable] = None,
        callback_on_step_end_tensor_inputs = ("latents",),
    ):
        """Generate images. ``uncond_skip_layers`` skips those zero-based blocks in the negative branch only (two
        forwards per step); None / () disables it. Unset: block 22, or ``reference_uncond_skip`` of the loader's
        checkpoint. ``uncond_skip_start`` / ``uncond_skip_end`` bound it in sigma-percent. ``slg_layers`` with ``slg_scale > 0`` adds a third, skipped positive forward and the
        fp32 correction ``slg_scale * (cond - cond_skip)``. Both are also gated by
        ``slg_start <= step_index / num_steps < slg_stop``."""
        self._interrupt = False
        depth = len(self.transformer.layers)
        if uncond_skip_layers is _UNSET:
            identity = getattr(self, "_unsloth_checkpoint", None)
            if identity is None:
                uncond_skip_layers, policy_end = DEFAULT_UNCOND_SKIP_LAYERS, 1.0
            else:
                uncond_skip_layers, policy_end = reference_uncond_skip(identity)
            if uncond_skip_end is None:
                uncond_skip_end = policy_end
        if uncond_skip_end is None:
            uncond_skip_end = 1.0
        if not 0 <= uncond_skip_start <= uncond_skip_end <= 1:
            raise ValueError("Require 0 <= uncond_skip_start <= uncond_skip_end <= 1.")
        uncond_skip = _validate_skip_layers(uncond_skip_layers, depth, "uncond_skip_layers")
        slg = _validate_skip_layers(slg_layers, depth, "slg_layers")
        # Tuples, not frozensets: a CUDA graph keys plain values, not objects.
        uncond_skip_kw = tuple(sorted(uncond_skip))
        slg_kw = tuple(sorted(slg))
        if not math.isfinite(slg_scale) or slg_scale < 0:
            raise ValueError("slg_scale must be finite and nonnegative.")
        if not 0 <= slg_start <= slg_stop <= 1:
            raise ValueError("Require 0 <= slg_start <= slg_stop <= 1.")
        if height <= 0 or width <= 0 or height % 16 or width % 16:
            raise ValueError("height and width must be positive multiples of 16.")
        if output_type not in {"pil", "np", "pt", "latent"}:
            raise ValueError("output_type must be pil, np, pt or latent.")
        if num_images_per_prompt < 1 or not math.isfinite(guidance_scale) or guidance_scale < 0:
            raise ValueError("Invalid image count or guidance scale.")
        if not 1 <= max_sequence_length <= 4096:
            raise ValueError("max_sequence_length must be between 1 and 4096.")
        if (prompt is None) == (prompt_embeds is None):
            raise ValueError("Provide exactly one of prompt and prompt_embeds.")
        device = self._execution_device
        if prompt_embeds is None:
            prompt_embeds, prompt_attention_mask = self.encode_prompt(
                prompt, max_sequence_length, device
            )
        batch = prompt_embeds.shape[0]

        def prepare(embeds, mask):
            if embeds.ndim != 3 or embeds.shape[0] != batch:
                raise ValueError("Prompt embeddings must be [B, S, C] with matching batch size.")
            if mask is None:
                mask = torch.ones(embeds.shape[:2], dtype = torch.bool, device = embeds.device)
            return (
                embeds.to(device).repeat_interleave(num_images_per_prompt, 0),
                mask.to(device).repeat_interleave(num_images_per_prompt, 0),
            )

        positive, positive_mask = prepare(prompt_embeds, prompt_attention_mask)
        do_cfg = guidance_scale != 1.0
        if do_cfg:
            if negative_prompt_embeds is None:
                negative_prompt = "" if negative_prompt is None else negative_prompt
                negatives = (
                    [negative_prompt] * batch
                    if isinstance(negative_prompt, str)
                    else list(negative_prompt)
                )
                if len(negatives) != batch:
                    raise ValueError("negative_prompt must match prompt batch size.")
                negative_prompt_embeds, negative_prompt_attention_mask = self.encode_prompt(
                    negatives, max_sequence_length, device
                )
            negative, negative_mask = prepare(
                negative_prompt_embeds, negative_prompt_attention_mask
            )
        shape = (
            batch * num_images_per_prompt,
            self.transformer.config.latent_channels,
            height // 16,
            width // 16,
        )
        if latents is None:
            latents = comfy_noise(shape, generator, device)
        elif tuple(latents.shape) != shape:
            raise ValueError(f"latents must have shape {shape}.")
        else:
            latents = latents.to(device = device, dtype = torch.float32)

        refine = not _env_off(TEXT_CACHE_ENV) and not _offloaded(self.transformer)
        cond_kw = self._branch(positive, positive_mask, refine)
        uncond_kw = self._branch(negative, negative_mask, refine) if do_cfg else None

        if sigmas is not None:
            self.scheduler.set_timesteps(device = device, sigmas = list(sigmas))
        else:
            self.scheduler.set_timesteps(num_inference_steps, device = device)
        timesteps = self.scheduler.timesteps
        num_steps = len(timesteps)
        uncond_window = None
        if uncond_skip_kw and (uncond_skip_start > 0 or uncond_skip_end < 1):
            shift = float(getattr(self.scheduler.config, "shift", 5.0))
            high = percent_to_sigma(uncond_skip_start, shift)
            low = percent_to_sigma(uncond_skip_end, shift)
            uncond_window = [
                low <= sigma <= high for sigma in self.scheduler.sigmas[:num_steps].tolist()
            ]
        for step_index, timestep in enumerate(self.progress_bar(timesteps)):
            if self._interrupt:
                continue
            skip_active = slg_start <= step_index / num_steps < slg_stop
            uncond_skip_active = skip_active and (
                uncond_window is None or uncond_window[step_index]
            )
            # CFG combines fp32 predictions, even when the DiT runs in bf16.
            # return_dict=False: a CUDA-graph replay hands back a tuple, never a dataclass.
            prediction = self.transformer(latents, timestep, **cond_kw, return_dict = False)[
                0
            ].float()
            conditional = prediction
            if do_cfg:
                skip_kwargs = (
                    {"skip_layers": uncond_skip_kw} if uncond_skip_kw and uncond_skip_active else {}
                )
                uncond = self.transformer(
                    latents, timestep, **uncond_kw, **skip_kwargs, return_dict = False
                )[0].float()
                prediction = uncond + guidance_scale * (prediction - uncond)
            if slg_kw and slg_scale > 0 and skip_active:
                skipped = self.transformer(
                    latents, timestep, **cond_kw, skip_layers = slg_kw, return_dict = False
                )[0].float()
                if not do_cfg:
                    prediction = skipped
                prediction = prediction + slg_scale * (conditional - skipped)
            latents = self.scheduler.step(prediction, timestep, latents, return_dict = False)[0]
            if callback_on_step_end is not None:
                # A loop, not a comprehension: before Python 3.12 a comprehension's locals() is its own scope.
                callback_kwargs = {}
                for name in callback_on_step_end_tensor_inputs:
                    callback_kwargs[name] = locals()[name]
                callback_outputs = callback_on_step_end(self, step_index, timestep, callback_kwargs)
                if callback_outputs is not None:
                    latents = callback_outputs.pop("latents", latents)
        if output_type == "latent":
            images = latents
        else:
            if self.vae is None:
                raise ValueError("A FLUX.2 VAE is required for decoded output.")
            images = self.vae_decode(latents)
            images = self.image_processor.postprocess(images, output_type = output_type)
        self.maybe_free_model_hooks()
        from diffusers.pipelines.pipeline_utils import ImagePipelineOutput

        return ImagePipelineOutput(images = images) if return_dict else (images,)

    def vae_decode(self, latents):
        """[B, 128, h, w] transformer latents -> [B, 3, 16h, 16w] in [-1, 1]: undo the VAE BatchNorm statistics, unpack
        the 2x2 patches to 32 channels, decode in the VAE's dtype (fp32 by default)."""
        z = latents.to(dtype = self.vae.dtype)
        bn = self.vae.bn
        eps = self.vae.config.batch_norm_eps
        z = z * (bn.running_var + eps).sqrt().to(z).view(1, -1, 1, 1) + bn.running_mean.to(z).view(
            1, -1, 1, 1
        )
        b, c, h, w = z.shape
        z = (
            z.reshape(b, c // 4, 2, 2, h, w)
            .permute(0, 1, 4, 2, 5, 3)
            .reshape(b, c // 4, 2 * h, 2 * w)
        )
        return self.vae.decode(z, return_dict = False)[0].clamp(-1, 1)


def transformers_version_ok() -> tuple[bool, Optional[str]]:
    """``(new enough, installed version)`` for the Qwen3.5 text encoder."""
    try:
        from importlib.metadata import version
        installed = version("transformers")
    except Exception:  # noqa: BLE001 - unreadable metadata: let the import decide
        return True, None
    try:
        from packaging.version import Version
        return Version(Version(installed).base_version) >= Version(MIN_TRANSFORMERS), installed
    except Exception:  # noqa: BLE001
        return True, installed


def transformers_refusal(installed: Optional[str]) -> str:
    return (
        f"MOSS-Image needs transformers >= {MIN_TRANSFORMERS} for its Qwen3.5 text encoder "
        f"(transformers.Qwen3_5Model), but this environment has transformers {installed or 'unknown'}. "
        "Update Unsloth Studio (or run: pip install -U 'transformers>=5.3') and restart it."
    )


def assert_transformers_supported() -> None:
    ok, installed = transformers_version_ok()
    if not ok:
        raise ValueError(transformers_refusal(installed))
    try:
        from transformers import Qwen3_5Model  # noqa: F401
    except Exception:  # noqa: BLE001
        raise ValueError(transformers_refusal(installed)) from None


def _live_cache_dir() -> Optional[str]:
    try:
        from utils.hf_cache_settings import active_hf_hub_cache
        return active_hf_hub_cache()
    except Exception:  # noqa: BLE001 - outside the Studio tree the hub default applies
        return None


def _tokenizer_class(repo_id: str, common: dict[str, Any]) -> Any:
    """The tokenizer class ``tokenizer/tokenizer_config.json`` names (Qwen2Tokenizer), else ``AutoTokenizer``.
    transformers 5.x's ``AutoTokenizer`` on a repo id first reads a ROOT ``config.json``, which these repos do not
    ship, so an offline load from the cache failed before reaching the tokenizer files."""
    import json

    import transformers

    try:
        if os.path.isdir(str(repo_id)):
            path = os.path.join(str(repo_id), "tokenizer", "tokenizer_config.json")
        else:
            from huggingface_hub import hf_hub_download
            path = hf_hub_download(
                repo_id,
                "tokenizer_config.json",
                subfolder = "tokenizer",
                cache_dir = common.get("cache_dir"),
                token = common.get("token"),
                local_files_only = bool(common.get("local_files_only")),
            )
        with open(path, encoding = "utf-8") as fh:
            name = json.load(fh).get("tokenizer_class")
    except Exception:  # noqa: BLE001 - AutoTokenizer reports the real error
        name = None
    cls = getattr(transformers, str(name), None) if name else None
    return cls if isinstance(cls, type) else transformers.AutoTokenizer


def drop_vision_tower(text_encoder: Any) -> bool:
    """Free Qwen3.5's vision tower (~0.4 B params): MOSS feeds the encoder text only, and the text forward never touches
    it (images enter only through ``pixel_values``). True when it was dropped."""
    visual = getattr(text_encoder, "visual", None)
    if not isinstance(visual, nn.Module):
        return False
    try:
        text_encoder.visual = None
    except Exception:  # noqa: BLE001 - a frozen module keeps its tower
        return False
    return True


def _conv_offload_hook(conv: Any) -> Any:
    registry = getattr(conv, "_diffusers_hook", None)
    get = getattr(registry, "get_hook", None)
    return get("group_offloading") if callable(get) else None


def _gdn_conv_onload(module: Any, args: Any) -> None:
    hook = _conv_offload_hook(module.conv1d)
    if hook is None:
        return
    hook.pre_forward(module.conv1d)
    stream = getattr(getattr(hook, "group", None), "stream", None)
    if stream is not None:
        stream.synchronize()


def _gdn_conv_offload(module: Any, args: Any, output: Any) -> None:
    hook = _conv_offload_hook(module.conv1d)
    if hook is not None:
        hook.post_forward(module.conv1d, None)


def install_gdn_offload_guard(text_encoder: Any) -> int:
    """Qwen3.5's gated-delta layers read ``self.conv1d.weight`` straight into the causal-conv kernel and never call
    ``conv1d``, so a leaf-level group offload (Studio's text-encoder streaming) never onloads that weight. Hooks on each
    gated-delta layer onload its conv for the layer's forward and offload it after; without a group-offload hook on the
    conv they do nothing. Returns the number of layers guarded."""
    count = 0
    for module in text_encoder.modules():
        if not type(module).__name__.endswith("GatedDeltaNet") or not isinstance(
            getattr(module, "conv1d", None), nn.Module
        ):
            continue
        if getattr(module, "_unsloth_conv_guard", False):
            continue
        module.register_forward_pre_hook(_gdn_conv_onload)
        module.register_forward_hook(_gdn_conv_offload)
        module._unsloth_conv_guard = True
        count += 1
    return count


def load_moss_image_text_encoder(source: str, dtype, **common):
    from transformers import Qwen3_5Model

    text_encoder = Qwen3_5Model.from_pretrained(
        source, subfolder = "text_encoder", dtype = dtype, **common
    )
    drop_vision_tower(text_encoder)
    install_gdn_offload_guard(text_encoder)
    return text_encoder.eval()


def load_moss_image_pipeline(
    repo_id: str,
    dtype,
    hf_token: Optional[str] = None,
    transformer: Any = None,
    text_encoder: Any = None,
    local_files_only: bool = False,
    check_cancelled: Optional[Callable[[], None]] = None,
    vae_dtype: Any = None,
    checkpoint: Optional[str] = None,
    vae: Any = None,
) -> MOSSImagePipeline:
    """A ready ``MOSSImagePipeline`` for ``repo_id`` (a repo id or a local diffusers folder), on CPU.

    Loads safetensors and JSON only, per subfolder; the repo's ``pipeline_moss_image.py`` is never imported.
    ``transformer`` / ``text_encoder`` let a quantised or pre-cast build hand in a component. The VAE decodes in fp32
    unless ``vae_dtype`` says otherwise (the reference and ComfyUI's MOSS launcher both decode in fp32). ``checkpoint``
    (else ``repo_id``) picks SFT or Base skip defaults."""
    check_cancelled = check_cancelled or (lambda: None)
    check_cancelled()
    assert_transformers_supported()
    import diffusers

    common: dict[str, Any] = {"local_files_only": local_files_only}
    cache_dir = _live_cache_dir()
    if cache_dir and not os.path.isdir(str(repo_id)):
        common["cache_dir"] = cache_dir
    if hf_token:
        common["token"] = hf_token
    tokenizer = _tokenizer_class(repo_id, common).from_pretrained(
        repo_id, subfolder = "tokenizer", **common
    )
    check_cancelled()
    if text_encoder is None:
        text_encoder = load_moss_image_text_encoder(repo_id, dtype, **common)
        check_cancelled()
    else:
        # A pre-cast build (the hosted int8 ConvRot encoder) carries the vision tower and needs the same guard.
        drop_vision_tower(text_encoder)
        install_gdn_offload_guard(text_encoder)
    if transformer is None:
        transformer = MOSSImageTransformer.from_pretrained(
            repo_id, subfolder = "transformer", torch_dtype = dtype, **common
        )
        check_cancelled()
    transformer.eval()
    if vae is None:
        vae = diffusers.AutoencoderKLFlux2.from_pretrained(
            repo_id, subfolder = "vae", torch_dtype = vae_dtype or torch.float32, **common
        )
    else:
        vae = vae.to(vae_dtype or torch.float32)
    check_cancelled()
    scheduler = MOSSImageScheduler.from_pretrained(repo_id, subfolder = "scheduler", **common)
    logger.info("diffusion.moss_image: assembled pipeline from %s per component", repo_id)
    pipe = MOSSImagePipeline(
        transformer = transformer,
        scheduler = scheduler,
        text_encoder = text_encoder,
        tokenizer = tokenizer,
        vae = vae,
    )
    pipe._unsloth_checkpoint = str(checkpoint or repo_id)
    return pipe


def in_tree_class(name: str) -> Optional[type]:
    """The in-tree class for a diffusers-style class name this module defines, else None."""
    return {
        MOSS_IMAGE_PIPELINE_CLASS: MOSSImagePipeline,
        MOSS_IMAGE_TRANSFORMER_CLASS: MOSSImageTransformer,
        MOSS_IMAGE_SCHEDULER_CLASS: MOSSImageScheduler,
    }.get(name)