Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

64.9 KB
Raw Blame
# Copyright (c) 2023 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import importlib
import re
import sys
from collections.abc import Callable
from dataclasses import dataclass, field
from types import SimpleNamespace
from typing import Any

import torch

from auto_round.export.formats import OutputFormat
from auto_round.modeling.fused_moe.replace_modules import apply_replacements, release_original_module_
from auto_round.utils import is_moe_model_via_config, logger, torch_version_at_least

mllms_with_limited_bs = (
    "llava",
    "qwen2_vl",
    "phi3_v",
    "mllama",
    "qwen2_5_omni",
    "qwen3_omni_moe",
    "glm_image",
    "mimo_audio",
    "qwen3_tts",
)  # Limitations on batch_size

SUPPORT_ONLY_TEXT_MODELS = [
    "phi3_v",
    "cogvlm2",
    "llava",
    "qwen2_vl",
    "qwen2_5_vl",
    "deepseek_vl_v2",
    "chatglm",
    "idefics3",
    "llama4",
    "internvl_chat",
    "glm4v_moe",
    "glm_image",
    "qwen3_vl_moe",
    "qwen2_5_omni",
    "qwen3_omni_moe",
    "gemma3",
    "bagel",
    "mimo_audio",
    "qwen3_tts",
]

NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]

# Maps architecture class names to virtual model_type keys.
# Used when config.model_type doesn't uniquely identify the model (e.g. MiMo-Audio).
from auto_round.utils.model import ARCHITECTURE_MODEL_TYPE_MAP, resolve_model_type  # noqa: E402

SPECIAL_SHARED_CACHE_KEYS = {
    "Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")
}
SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)
SPECIAL_SHARED_CACHE_KEYS["StableAudioDiTModel"] = ("encoder_hidden_states",)
SPECIAL_SHARED_CACHE_KEYS["Gemma4ForConditionalGeneration"] = ("position_ids",)
SPECIAL_SHARED_CACHE_KEYS["WanTransformer3DModel"] = ("rotary_emb",)
MISTRAL_3_2_MODELS = ["Mistral-Small-3.2", "Magistral-Small", "Devstral-Small"]


def _normalize_gemma4_per_layer_input(positional_inputs, hidden_states):
    if positional_inputs is None or len(positional_inputs) == 0:
        return positional_inputs

    per_layer_input = positional_inputs[0]
    if not isinstance(per_layer_input, torch.Tensor) or per_layer_input.shape[1] == hidden_states.shape[1]:
        return positional_inputs

    hs_seq = hidden_states.shape[1]
    pl_seq = per_layer_input.shape[1]
    if hs_seq <= pl_seq:
        per_layer_input = per_layer_input[:, :hs_seq, :]
    else:
        pad = per_layer_input[:, -1:, :].expand(-1, hs_seq - pl_seq, -1)
        per_layer_input = torch.cat([per_layer_input, pad], dim=1)

    normalized_inputs = list(positional_inputs)
    normalized_inputs[0] = per_layer_input
    return type(positional_inputs)(normalized_inputs) if isinstance(positional_inputs, tuple) else normalized_inputs


def prepare_special_model_block_inputs(block, rotary_input, input_others, positional_inputs=None):
    """Rewrite replay inputs for blocks that need model-specific handling."""

    # Guard: ensure position_ids is a tensor, not a list or None.
    if "position_ids" in input_others:
        pid = input_others["position_ids"]
        if isinstance(pid, list):
            if len(pid) == 1:
                input_others["position_ids"] = pid[0]
            elif len(pid) == 0:
                input_others["position_ids"] = (
                    torch.arange(rotary_input.shape[1], device=rotary_input.device, dtype=torch.long)
                    .unsqueeze(0)
                    .expand(rotary_input.shape[0], -1)
                )
        elif pid is None:
            input_others["position_ids"] = (
                torch.arange(rotary_input.shape[1], device=rotary_input.device, dtype=torch.long)
                .unsqueeze(0)
                .expand(rotary_input.shape[0], -1)
            )

    special_replay_type = getattr(block, "_autoround_special_replay", None)
    if special_replay_type == "gemma4" or special_replay_type == "gemma4_unified":
        prepared_inputs = _prepare_gemma4_replay_inputs(
            block,
            rotary_input,
            position_ids=input_others.get("position_ids"),
            position_embeddings=input_others.get("position_embeddings"),
            attention_mask=input_others.get("attention_mask"),
            shared_kv_states=input_others.get("shared_kv_states"),
            past_key_values=input_others.get("past_key_values"),
            config=getattr(block, "_gemma4_config_ref", None),
        )
        for key, value in prepared_inputs.items():
            if value is not None or key in input_others or key == "shared_kv_states":
                input_others[key] = value
        positional_inputs = _normalize_gemma4_per_layer_input(positional_inputs, rotary_input)
    return input_others, positional_inputs


def _get_gemma4_shared_kv_states_global(block):
    """Return the shared KV states dict for Gemma4 block-wise quantization."""
    ref = getattr(block, "_shared_kv_states_global_ref", None)
    if ref is not None:
        return ref
    return {}


def _get_gemma4_rotary_emb(block, default_rotary_emb=None):
    rotary_emb_ref = getattr(block, "_rotary_emb_ref", None)
    if rotary_emb_ref:
        return rotary_emb_ref[0]
    return getattr(block, "_rotary_emb", default_rotary_emb)


def _rebuild_gemma4_attention_mask(config, hidden_states, position_ids, layer_type, past_key_values=None):
    from transformers.masking_utils import create_causal_mask, create_sliding_window_causal_mask

    mask_builder = create_causal_mask if layer_type == "full_attention" else create_sliding_window_causal_mask
    return mask_builder(
        config=config,
        inputs_embeds=hidden_states,
        attention_mask=None,
        past_key_values=past_key_values,
        position_ids=position_ids,
    )


def _prepare_gemma4_replay_inputs(
    block,
    rotary_input,
    *,
    position_ids=None,
    position_embeddings=None,
    attention_mask=None,
    shared_kv_states=None,
    past_key_values=None,
    config=None,
    default_rotary_emb=None,
    default_shared_kv_states=None,
):
    attn = getattr(block, "self_attn", None)
    layer_type = getattr(attn, "layer_type", None)
    head_dim = getattr(attn, "head_dim", None)

    if attn is not None and hasattr(attn, "store_full_length_kv") and shared_kv_states is None:
        if default_shared_kv_states is not None:
            shared_kv_states = default_shared_kv_states
        else:
            shared_kv_states = _get_gemma4_shared_kv_states_global(block)
            if getattr(block, "layer_idx", None) == 0:
                shared_kv_states.clear()

    need_position_embeddings = position_embeddings is None
    if isinstance(position_embeddings, dict):
        cached_position_embeddings = position_embeddings.get(layer_type) if layer_type is not None else None
        need_position_embeddings = cached_position_embeddings is None
    else:
        cached_position_embeddings = position_embeddings

    if (
        not need_position_embeddings
        and head_dim is not None
        and isinstance(cached_position_embeddings, (tuple, list))
        and cached_position_embeddings
    ):
        need_position_embeddings = cached_position_embeddings[0].shape[-1] != head_dim

    if need_position_embeddings and layer_type is not None and position_ids is not None:
        rotary_emb = _get_gemma4_rotary_emb(block, default_rotary_emb)
        if rotary_emb is not None:
            rebuilt_position_embeddings = rotary_emb(rotary_input, position_ids, layer_type)
            if isinstance(position_embeddings, dict):
                position_embeddings = dict(position_embeddings)
                position_embeddings[layer_type] = rebuilt_position_embeddings
            else:
                position_embeddings = rebuilt_position_embeddings

    if config is not None and layer_type is not None and position_ids is not None:
        try:
            attention_mask = _rebuild_gemma4_attention_mask(
                config,
                hidden_states=rotary_input,
                position_ids=position_ids,
                layer_type=layer_type,
                past_key_values=past_key_values,
            )
        except Exception:
            pass

    return {
        "position_embeddings": position_embeddings,
        "attention_mask": attention_mask,
        "shared_kv_states": shared_kv_states,
    }


def _patch_gemma4_model(model):
    """Patch each Gemma4 decoder layer so it recomputes position_embeddings and
    attention_mask from the cached position_ids when the cached versions have
    wrong dimensions (sliding_attention vs full_attention head_dims differ).

    During auto-round block-wise quantization the cached inputs from block 0
    (always a sliding_attention layer) are reused for every subsequent block.
    Full-attention layers (head_dim=512) would receive position embeddings
    computed for sliding layers (head_dim=256), causing a shape mismatch crash.
    """
    import types as _types

    try:
        from transformers.models.gemma4.modeling_gemma4 import Gemma4TextModel
    except ImportError:
        return model

    # Find the Gemma4TextModel by traversing the hierarchy
    text_model = None
    for _, submodule in model.named_modules():
        if isinstance(submodule, Gemma4TextModel):
            text_model = submodule
            break

    if text_model is None:
        return model

    rotary_emb = text_model.rotary_emb
    # Shared dict to propagate KV state between anchor/sharer layers (like
    # Gemma4TextModel.forward does in newer transformers versions).
    shared_kv_states_global = {}

    for layer in text_model.layers:
        object.__setattr__(layer, "_autoround_special_replay", "gemma4")
        object.__setattr__(layer, "_gemma4_config_ref", text_model.config)
        original_forward = layer.forward
        layer_type = getattr(getattr(layer, "self_attn", None), "layer_type", None)
        if layer_type is None:
            continue
        head_dim = getattr(getattr(layer, "self_attn", None), "head_dim", None)
        is_full = layer_type == "full_attention"

        def _make_layer_forward(orig_fwd, lt, hd, is_full_attn, re, cfg, skv_global):
            import inspect

            orig_params = list(inspect.signature(orig_fwd).parameters)
            orig_has_shared_kv = "shared_kv_states" in orig_params

            def patched_layer_forward(
                self,
                hidden_states,
                per_layer_input=None,
                shared_kv_states=None,
                position_embeddings=None,
                attention_mask=None,
                position_ids=None,
                **kwargs,
            ):
                # Rebuild Gemma4 layer-specific replay inputs from the minimal
                # shared cache so later layers do not need variable block inputs.
                if shared_kv_states is None and getattr(self, "layer_idx", None) == 0:
                    skv_global.clear()
                prepared_inputs = _prepare_gemma4_replay_inputs(
                    self,
                    hidden_states,
                    position_ids=position_ids,
                    position_embeddings=position_embeddings,
                    attention_mask=attention_mask,
                    shared_kv_states=shared_kv_states,
                    past_key_values=kwargs.get("past_key_values"),
                    config=cfg,
                    default_rotary_emb=re,
                    default_shared_kv_states=skv_global,
                )
                position_embeddings = prepared_inputs["position_embeddings"]
                attention_mask = prepared_inputs["attention_mask"]
                shared_kv_states = prepared_inputs["shared_kv_states"]

                # per_layer_input is token-specific but is cached as shared positional
                # input (only 1st batch stored). Truncate/pad to match hidden_states seq_len.
                if per_layer_input is not None and per_layer_input.shape[1] != hidden_states.shape[1]:
                    hs_seq = hidden_states.shape[1]
                    pl_seq = per_layer_input.shape[1]
                    if hs_seq <= pl_seq:
                        per_layer_input = per_layer_input[:, :hs_seq, :]
                    else:
                        pad = per_layer_input[:, -1:, :].expand(-1, hs_seq - pl_seq, -1)
                        per_layer_input = torch.cat([per_layer_input, pad], dim=1)

                if orig_has_shared_kv:
                    return orig_fwd(
                        hidden_states,
                        per_layer_input,
                        shared_kv_states,
                        position_embeddings=position_embeddings,
                        attention_mask=attention_mask,
                        position_ids=position_ids,
                        **kwargs,
                    )
                return orig_fwd(
                    hidden_states,
                    per_layer_input,
                    position_embeddings=position_embeddings,
                    attention_mask=attention_mask,
                    position_ids=position_ids,
                    **kwargs,
                )

            return patched_layer_forward

        layer.forward = _types.MethodType(
            _make_layer_forward(
                original_forward, layer_type, head_dim, is_full, rotary_emb, text_model.config, shared_kv_states_global
            ),
            layer,
        )

    return model


def _handle_special_model(model):
    model_type = resolve_model_type(model)
    if model_type == "deepseek_vl_v2":
        from functools import partial

        model.forward = partial(_deepseek_vl2_forward, model)
    if model_type == "qwen2_5_omni":
        from functools import partial

        model.forward = partial(_qwen2_5_omni_forward, model)
    if model_type == "qwen3_omni_moe":
        from functools import partial

        model.forward = partial(_qwen3_omni_moe_forward, model)
    if model_type == "qwen3_tts":
        from functools import partial

        model.forward = partial(_qwen3_tts_forward, model)
    if model_type == "mimo_audio":
        from functools import partial

        model.forward = partial(_mimo_audio_forward, model)
    if hasattr(model, "config") and (model_type == "gemma4"):
        import transformers
        from packaging import version

        if version.parse(transformers.__version__) < version.parse("5.6"):
            _patch_gemma4_model(model)
        else:
            _attach_gemma4_rotary_emb(model)
        logger.warning(
            "Applying a monkey patch to Gemma4 to reduce RAM usage. "
            "This patch has only been validated with limited Transformers versions. "
            "Proceed with caution."
        )
    if hasattr(model, "config") and model_type == "gemma4_unified":
        _attach_gemma4_unified_rotary_emb(model)
    return model


def update_module(
    model, formats: list[OutputFormat] | None = None, trust_remote_code: bool = True, cleanup_original: bool = True
):
    gguf_export = formats is not None and any(format_.is_gguf() for format_ in formats)
    model = apply_replacements(model, gguf_export=gguf_export)

    if cleanup_original:
        release_original_module_(model)

    return model


def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):
    model.forward = model.language.forward
    block_names = []
    if quant_vision:
        block_names.append([f"vision.blocks.{i}" for i in range(len(model.vision.blocks))])
        block_names.append([f"projector.layers.{i}" for i in range(len(model.projector.layers))])
    block_names.append([f"language.model.layers.{i}" for i in range(len(model.language.model.layers))])
    return block_names


def _get_qwen2_5_omni_multimodal_block(model, quant_vision=False):
    """Get block names for Qwen2.5-Omni model.

    Qwen2.5-Omni has the following structure:
    - thinker: Contains audio_tower, visual, model (text decoder)
    - talker: Contains model (talker decoder)
    - token2wav: Audio decoder

    For quantization, we focus on:
    - thinker.model.layers (text decoder layers) - main LLM layers
    - Optionally: visual encoder blocks, audio encoder layers

    talker is excluded by default because quantizing it has been observed to
    degrade audio quality in long-form generation.
    """
    block_names = []

    # Quantize visual encoder blocks if quant_vision is enabled
    if quant_vision:
        if hasattr(model, "thinker") and hasattr(model.thinker, "visual") and hasattr(model.thinker.visual, "blocks"):
            block_names.append([f"thinker.visual.blocks.{i}" for i in range(len(model.thinker.visual.blocks))])
        if hasattr(model, "thinker") and hasattr(model.thinker, "audio_tower"):
            if hasattr(model.thinker.audio_tower, "layers"):
                block_names.append(
                    [f"thinker.audio_tower.layers.{i}" for i in range(len(model.thinker.audio_tower.layers))]
                )

    # Thinker text model layers (main LLM decoder)
    if hasattr(model, "thinker") and hasattr(model.thinker, "model") and hasattr(model.thinker.model, "layers"):
        block_names.append([f"thinker.model.layers.{i}" for i in range(len(model.thinker.model.layers))])

    return block_names


def _get_qwen3_omni_moe_multimodal_block(model, quant_vision=False):
    """Get block names for Qwen3-Omni MoE model.

    Qwen3-Omni has the following structure:
    - thinker.model: Contains visual/audio towers and the text decoder in newer Transformers releases
    - thinker: Contains visual/audio towers directly in older Transformers releases
    - talker: Contains model (talker decoder), code_predictor
    - code2wav: Audio decoder

    For quantization, we focus on:
    - thinker.model.layers (text decoder layers) - main LLM layers
    - Optionally: visual encoder blocks, audio encoder layers

    talker is excluded by default because quantizing it has been observed to
    degrade audio quality in long-form generation .
    """
    block_names = []
    thinker = getattr(model, "thinker", None)
    thinker_model = getattr(thinker, "model", None)

    # Quantize visual encoder blocks if quant_vision is enabled
    if quant_vision:
        visual = getattr(thinker_model, "visual", None)
        visual_blocks = getattr(visual, "blocks", None)
        visual_prefix = "thinker.model.visual"
        if visual_blocks is None or len(visual_blocks) == 0:
            visual = getattr(thinker, "visual", None)
            visual_blocks = getattr(visual, "blocks", None)
            visual_prefix = "thinker.visual"
        if visual_blocks is not None:
            block_names.append([f"{visual_prefix}.blocks.{i}" for i in range(len(visual_blocks))])

        audio_tower = getattr(thinker_model, "audio_tower", None)
        audio_layers = getattr(audio_tower, "layers", None)
        audio_prefix = "thinker.model.audio_tower"
        if audio_layers is None or len(audio_layers) == 0:
            audio_tower = getattr(thinker, "audio_tower", None)
            audio_layers = getattr(audio_tower, "layers", None)
            audio_prefix = "thinker.audio_tower"
        if audio_layers is not None:
            block_names.append([f"{audio_prefix}.layers.{i}" for i in range(len(audio_layers))])

    # Thinker text model layers (main LLM decoder). Transformers may wrap the
    # decoder in `language_model` in newer Qwen3-Omni releases.
    layer_prefix, layers = _get_qwen3_omni_thinker_layer_info(model)
    if layers is not None:
        block_names.append([f"{layer_prefix}.{i}" for i in range(len(layers))])

    return block_names


def _get_qwen3_omni_thinker_layer_info(model):
    """Return the module path and layers for either supported thinker layout."""
    thinker_model = getattr(getattr(model, "thinker", None), "model", None)
    if thinker_model is None:
        return None, None

    if hasattr(thinker_model, "layers"):
        return "thinker.model.layers", thinker_model.layers

    language_model = getattr(thinker_model, "language_model", None)
    if language_model is not None and hasattr(language_model, "layers"):
        return "thinker.model.language_model.layers", language_model.layers

    return None, None


def _get_glm_image_multimodal_block(model, quant_vision=False):
    """Get block names for GLM-Image AR model.

    GLM-Image AR model structure:
    - model.visual.blocks: vision encoder
    - model.language_model.layers: autoregressive text backbone

    By default, only text backbone is quantized. Set quant_vision=True to include
    the visual encoder blocks.
    """
    block_names = []

    if quant_vision and hasattr(model, "model") and hasattr(model.model, "visual"):
        if hasattr(model.model.visual, "blocks"):
            block_names.append([f"model.visual.blocks.{i}" for i in range(len(model.model.visual.blocks))])

    if hasattr(model, "model") and hasattr(model.model, "language_model"):
        if hasattr(model.model.language_model, "layers"):
            block_names.append(
                [f"model.language_model.layers.{i}" for i in range(len(model.model.language_model.layers))]
            )

    return block_names


def _get_mimo_audio_multimodal_block(model, quant_vision=False):
    """Get block names for MiMo-Audio model.

    MiMo-Audio (MiMoAudioForCausalLM) has the following structure:
    - model.model.layers: Main Qwen2-based LLM decoder (28 layers)
    - input_local_transformer.layers: Input audio encoder (6 layers)
    - local_transformer.layers: Audio decoder / patch decoder (16 layers)

    Currently only the main LLM backbone is quantized because:
    - Audio encoder/decoder require audio calibration data (not yet supported)
    - Audio modules use smaller hidden dim (1024 vs 4096) with limited quantization benefit

    Args:
        model: The MiMoAudioForCausalLM or MiMoAudioModel instance.
    """
    block_names = []

    # Main LLM decoder layers (backbone - always quantized)
    if hasattr(model, "model") and hasattr(model.model, "layers"):
        # MiMoAudioForCausalLM: layers at model.model.layers
        block_names.append([f"model.layers.{i}" for i in range(len(model.model.layers))])
    elif hasattr(model, "layers"):
        # MiMoAudioModel (base): layers at model.layers directly
        block_names.append([f"layers.{i}" for i in range(len(model.layers))])

    return block_names


def _get_qwen3_tts_multimodal_block(model, quant_vision=False):
    """Get block names for Qwen3-TTS model.

    Qwen3-TTS (Qwen3TTSForConditionalGeneration) has a talker backbone with the
    following possible module layouts depending on the modeling code version:
    - tts_model.model.layers: Main TTS transformer decoder layers
    - talker.model.layers: Alternative attribute name (matches talker_config)
    - model.model.layers: Standard fallback

    Only the talker backbone is quantized. Audio codec layers are not quantized
    because audio calibration data is not yet supported.

    Args:
        model: The Qwen3TTSForConditionalGeneration model instance.
        quant_vision: Unused. Accepted for interface compatibility with
            SPECIAL_MULTIMODAL_BLOCK registry.
    """
    block_names = []
    # Try tts_model.model.layers
    if hasattr(model, "tts_model") and hasattr(getattr(model.tts_model, "model", None), "layers"):
        block_names.append([f"tts_model.model.layers.{i}" for i in range(len(model.tts_model.model.layers))])
    # Try talker.model.layers (alternative attr name from talker_config)
    if not block_names and hasattr(model, "talker") and hasattr(getattr(model.talker, "model", None), "layers"):
        block_names.append([f"talker.model.layers.{i}" for i in range(len(model.talker.model.layers))])
    # Fallback: model.model.layers (standard structure)
    if not block_names and hasattr(model, "model") and hasattr(model.model, "layers"):
        block_names.append([f"model.layers.{i}" for i in range(len(model.model.layers))])
    return block_names


def _get_bagel_multimodal_block(model, quant_vision=False):
    """Get block names for BAGEL MoT (Mixture of Transformers) model.

    BAGEL model structure:
    - language_model.model.layers: Qwen2-based LLM with MoT dual paths
    - vit_model: SigLIP vision encoder (not quantized by default)
    - connector: Vision-language MLP connector
    - encoder/decoder: VAE autoencoder
    - time_embedder, vae2llm, llm2vae: bridge modules

    By default, only the language_model layers are quantized.
    """
    block_names = []

    if hasattr(model, "language_model") and hasattr(model.language_model, "model"):
        if hasattr(model.language_model.model, "layers"):
            block_names.append(
                [f"language_model.model.layers.{i}" for i in range(len(model.language_model.model.layers))]
            )

    return block_names


SPECIAL_MULTIMODAL_BLOCK = {
    "deepseek_vl_v2": _get_deepseek_vl2_multimodal_block,
    "qwen2_5_omni": _get_qwen2_5_omni_multimodal_block,
    "qwen3_omni_moe": _get_qwen3_omni_moe_multimodal_block,
    "glm_image": _get_glm_image_multimodal_block,
    "mimo_audio": _get_mimo_audio_multimodal_block,
    "qwen3_tts": _get_qwen3_tts_multimodal_block,
    "bagel": _get_bagel_multimodal_block,
}


def _deepseek_vl2_forward(
    model,
    input_ids=None,
    position_ids=None,
    attention_mask=None,
    past_key_values=None,
    inputs_embeds=None,
    images=None,
    images_seq_mask=None,
    images_spatial_crop=None,
    labels=None,
    use_cache=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
    cache_position=None,
    **kwargs,
):
    inputs_embeds = model.prepare_inputs_embeds(
        input_ids=input_ids,
        images=images,
        images_seq_mask=images_seq_mask,
        images_spatial_crop=images_spatial_crop,
    )
    return model.language(
        input_ids=None,
        attention_mask=attention_mask,
        position_ids=position_ids,
        past_key_values=past_key_values,
        inputs_embeds=inputs_embeds,
        labels=labels,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
        cache_position=cache_position,
    )


def _qwen2_5_omni_forward(
    model,
    input_ids=None,
    input_features=None,
    pixel_values=None,
    pixel_values_videos=None,
    image_grid_thw=None,
    video_grid_thw=None,
    attention_mask=None,
    feature_attention_mask=None,
    audio_feature_lengths=None,
    position_ids=None,
    past_key_values=None,
    inputs_embeds=None,
    rope_deltas=None,
    labels=None,
    use_cache=None,
    use_audio_in_video=None,
    cache_position=None,
    video_second_per_grid=None,
    **kwargs,
):
    """Forward function for Qwen2.5-Omni model.

    This delegates to the thinker module for calibration, then optionally
    runs a forward through the talker to ensure its layers are also calibrated.
    """
    thinker_output = model.thinker(
        input_ids=input_ids,
        input_features=input_features,
        pixel_values=pixel_values,
        pixel_values_videos=pixel_values_videos,
        image_grid_thw=image_grid_thw,
        video_grid_thw=video_grid_thw,
        attention_mask=attention_mask,
        feature_attention_mask=feature_attention_mask,
        audio_feature_lengths=audio_feature_lengths,
        position_ids=position_ids,
        past_key_values=past_key_values,
        inputs_embeds=inputs_embeds,
        rope_deltas=rope_deltas,
        labels=labels,
        use_cache=use_cache,
        use_audio_in_video=use_audio_in_video,
        cache_position=cache_position,
        video_second_per_grid=video_second_per_grid,
        output_hidden_states=True,
        return_dict=True,
        **kwargs,
    )

    # Run talker forward if available (for calibration purposes)
    if hasattr(model, "talker") and model.has_talker:
        thinker_hidden = thinker_output.hidden_states[-1] if thinker_output.hidden_states else None

        if thinker_hidden is not None:
            # ---- calibrate thinker_to_talker_proj (nn.Linear) ----
            thinker_embeds = model.thinker.get_input_embeddings()(input_ids)
            proj_dtype = next(model.talker.thinker_to_talker_proj.parameters()).dtype
            talker_inputs_embeds = model.talker.thinker_to_talker_proj(thinker_embeds.to(proj_dtype))

            # ---- calibrate talker decoder layers ----
            talker_dtype = next(model.talker.model.parameters()).dtype
            talker_output = model.talker.model(
                inputs_embeds=talker_inputs_embeds.to(talker_dtype),
                attention_mask=attention_mask,
                use_cache=False,
            )

            # ---- calibrate codec_head (nn.Linear) ----
            if hasattr(model.talker, "codec_head"):
                talker_last_hidden = (
                    talker_output[0]
                    if not hasattr(talker_output, "last_hidden_state")
                    else talker_output.last_hidden_state
                )
                _ = model.talker.codec_head(talker_last_hidden.to(talker_dtype))

    return thinker_output


def _qwen3_omni_moe_forward(
    model,
    input_ids=None,
    input_features=None,
    pixel_values=None,
    pixel_values_videos=None,
    image_grid_thw=None,
    video_grid_thw=None,
    attention_mask=None,
    feature_attention_mask=None,
    audio_feature_lengths=None,
    position_ids=None,
    past_key_values=None,
    inputs_embeds=None,
    rope_deltas=None,
    labels=None,
    use_cache=None,
    output_router_logits=None,
    use_audio_in_video=None,
    cache_position=None,
    video_second_per_grid=None,
    **kwargs,
):
    """Forward function for Qwen3-Omni-MoE model.

    This runs forward through both thinker and talker modules for calibration.
    The thinker processes text/vision/audio input, and talker uses thinker's
    hidden states to prepare for speech synthesis.

    In real inference the talker receives inputs built from two projections:
      - ``text_projection``: maps thinker word embeddings → talker hidden dim
        (used for pure-text tokens).
      - ``hidden_projection``: maps thinker intermediate hidden states → talker
        hidden dim (used for multimodal tokens).

    During calibration we exercise both projection paths and then run one
    forward pass through the talker decoder so that every linear layer
    (attention, MoE/MLP, codec_head) observes representative activations.
    """
    # Run thinker forward with output_hidden_states to get hidden states for talker
    thinker_output = model.thinker(
        input_ids=input_ids,
        input_features=input_features,
        pixel_values=pixel_values,
        pixel_values_videos=pixel_values_videos,
        image_grid_thw=image_grid_thw,
        video_grid_thw=video_grid_thw,
        attention_mask=attention_mask,
        feature_attention_mask=feature_attention_mask,
        audio_feature_lengths=audio_feature_lengths,
        position_ids=position_ids,
        past_key_values=past_key_values,
        inputs_embeds=inputs_embeds,
        rope_deltas=rope_deltas,
        labels=labels,
        use_cache=use_cache,
        output_router_logits=output_router_logits,
        use_audio_in_video=use_audio_in_video,
        cache_position=cache_position,
        video_second_per_grid=video_second_per_grid,
        output_hidden_states=True,
        return_dict=True,
        **kwargs,
    )

    # Run talker forward if available (for calibration purposes)
    if getattr(model, "has_talker", False) and getattr(model, "talker", None) is not None:
        thinker_hidden = thinker_output.hidden_states[-1] if thinker_output.hidden_states else None

        if thinker_hidden is not None:
            # ---- calibrate text_projection (ResizeMLP) ----
            thinker_embeds = model.thinker.get_input_embeddings()(input_ids)
            proj_dtype = next(model.talker.text_projection.parameters()).dtype
            talker_inputs_embeds = model.talker.text_projection(thinker_embeds.to(proj_dtype))

            # ---- calibrate hidden_projection (ResizeMLP) ----
            if hasattr(model.talker, "hidden_projection"):
                hidden_proj_dtype = next(model.talker.hidden_projection.parameters()).dtype
                _ = model.talker.hidden_projection(thinker_hidden.to(hidden_proj_dtype))

            # ---- calibrate talker decoder layers ----
            talker_dtype = next(model.talker.model.parameters()).dtype
            talker_output = model.talker.model(
                inputs_embeds=talker_inputs_embeds.to(talker_dtype),
                attention_mask=attention_mask,
                use_cache=False,
            )

            # ---- calibrate codec_head ----
            if hasattr(model.talker, "codec_head"):
                talker_last_hidden = (
                    talker_output[0]
                    if not hasattr(talker_output, "last_hidden_state")
                    else talker_output.last_hidden_state
                )
                _ = model.talker.codec_head(talker_last_hidden.to(talker_dtype))

    return thinker_output


def _mimo_audio_forward(
    model,
    input_ids=None,
    attention_mask=None,
    position_ids=None,
    past_key_values=None,
    inputs_embeds=None,
    labels=None,
    use_cache=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
    cache_position=None,
    **kwargs,
):
    """Forward function for MiMo-Audio model during calibration.

    MiMo-Audio's native forward expects 3D input_ids [B, audio_channels+1, T] with
    interleaved text and audio tokens. During calibration with text-only data, we
    route through the Qwen2Model backbone directly to calibrate the main decoder layers.
    """
    backbone = model.model

    if input_ids is not None and inputs_embeds is None:
        inputs_embeds = backbone.embed_tokens(input_ids)
        input_ids = None

    return backbone(
        input_ids=input_ids,
        attention_mask=attention_mask,
        position_ids=position_ids,
        past_key_values=past_key_values,
        inputs_embeds=inputs_embeds,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
        cache_position=cache_position,
    )


def _qwen3_tts_forward(
    model,
    input_ids=None,
    attention_mask=None,
    position_ids=None,
    past_key_values=None,
    inputs_embeds=None,
    labels=None,
    use_cache=None,
    output_attentions=None,
    output_hidden_states=None,
    return_dict=None,
    cache_position=None,
    **kwargs,
):
    """Forward function for Qwen3-TTS model.

    Qwen3-TTS uses a discrete multi-codebook LM architecture. During calibration
    we route the forward through the talker, converting text input_ids to
    inputs_embeds via text_embedding + text_projection.
    """
    tts_backbone = None
    for candidate in [getattr(model, "tts_model", None), getattr(model, "talker", None)]:
        if candidate is None:
            continue
        has_text_embed = hasattr(getattr(candidate, "model", None), "text_embedding")
        has_text_proj = hasattr(candidate, "text_projection")
        if has_text_embed and has_text_proj:
            tts_backbone = candidate
            break
    # Fallback: use whichever backbone exists even without text pathway
    if tts_backbone is None:
        tts_backbone = getattr(model, "tts_model", None) or getattr(model, "talker", None)

    if tts_backbone is not None:
        # Convert text input_ids to inputs_embeds through text pathway
        if input_ids is not None and inputs_embeds is None:
            text_embedding = getattr(tts_backbone.model, "text_embedding", None)
            text_projection = getattr(tts_backbone, "text_projection", None)
            if text_embedding is None or text_projection is None:
                raise RuntimeError(
                    "Qwen3-TTS backbone is missing text_embedding or text_projection. "
                    "Cannot convert text input_ids to inputs_embeds for calibration."
                )
            inputs_embeds = text_projection(text_embedding(input_ids))
            input_ids = None

        return tts_backbone(
            input_ids=input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            labels=labels,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            cache_position=cache_position,
            **kwargs,
        )
    # Fallback: model has standard forward
    return model.__class__.forward(
        model,
        input_ids=input_ids,
        attention_mask=attention_mask,
        position_ids=position_ids,
        past_key_values=past_key_values,
        inputs_embeds=inputs_embeds,
        labels=labels,
        use_cache=use_cache,
        output_attentions=output_attentions,
        output_hidden_states=output_hidden_states,
        return_dict=return_dict,
        cache_position=cache_position,
        **kwargs,
    )


def check_mllm_only_support_bs1(model: torch.nn.Module):
    """
    Checks model configuration to determine if it's necessary to limit bs to avoid potential input shape mismatches.
    """
    effective_type = resolve_model_type(model) or ""
    for key in mllms_with_limited_bs:
        if key in effective_type:
            return True
    return False


class ModelNameMatcher:
    """model.config.name_or_path"""

    def __init__(self, pattern: str, mode="in"):
        self.pattern = pattern
        self.mode = mode

    def __call__(self, model) -> bool:
        name = getattr(model.config, "name_or_path", "")
        if self.mode == "full":
            return name == self.pattern
        elif self.mode == "in":
            return self.pattern in name
        elif self.mode == "regex":
            return re.search(self.pattern, name) is not None
        else:
            raise ValueError("unsupported mode {self.mode}")

    """Matches config.architectures."""


class ArchitectureMatcher:
    """match config.architectures"""

    def __init__(self, arch: str, mode="in"):
        self.arch = arch
        self.mode = mode

    def __call__(self, model) -> bool:
        archs = getattr(model.config, "architectures", [])
        archs_str = ",".join(archs) if isinstance(archs, list) else str(archs)

        if self.mode == "full":
            return archs_str == self.arch
        elif self.mode == "in":
            return self.arch in archs_str
        elif self.mode == "regex":
            return re.search(self.arch, archs_str) is not None
        else:
            raise ValueError(f"unsupported mode {self.mode}")


class ModelTypeMatcher:
    """match config.architectures"""

    def __init__(self, model_type: str, mode="in"):
        self.model_type = model_type
        self.mode = mode

    def __call__(self, model) -> bool:
        model_type = getattr(model.config, "model_type", None)
        if model_type is None:
            return False

        if self.mode == "full":
            return model_type == self.model_type
        elif self.mode == "in":
            return self.model_type in model_type
        elif self.mode == "regex":
            return re.search(self.model_type, model_type) is not None
        else:
            raise ValueError(f"unsupported mode {self.mode}")


@dataclass
class PreDefinedIgnoreLayers:
    matchers: list[Callable[[Any], bool]]
    ignore_layers: list[str] = field(default_factory=list)


_PRE_DEFINED_IGNORE_LAYERS: list[PreDefinedIgnoreLayers] = []


def register_ignore_layers(
    matchers: list[Callable[[Any], bool]], ignore_layers: list[str | Callable[[torch.nn.Module], str | list[str]]]
):
    rule = PreDefinedIgnoreLayers(matchers, ignore_layers)
    _PRE_DEFINED_IGNORE_LAYERS.append(rule)


# longcat
register_ignore_layers(
    matchers=[
        ArchitectureMatcher(r"Longcat", mode="in"),
    ],
    ignore_layers=[
        "classifier",  # transforms directly call the weights of this layer
    ],
)


def get_glm_flash_ignore_layers(model) -> list[str]:
    num_dense_layer = 1
    if hasattr(model, "config") and hasattr(model.config, "first_k_dense_replace"):
        num_dense_layer = model.config.first_k_dense_replace
    ignore_layers = [f"layers.{i}.mlp" for i in range(num_dense_layer)]
    return ignore_layers


# glmflash
register_ignore_layers(
    matchers=[
        ArchitectureMatcher(r"Glm4MoeLite", mode="in"),
    ],
    ignore_layers=[
        get_glm_flash_ignore_layers,  # vllm issue
    ],
)

# GLM-5.3-Flash family: ``glm_moe_dsa`` (text) and ``glm5_next`` (multimodal).
#
# Both keep their whole DSA indexer out of quantization:
#   * ``{GlmMoeDsa,Glm5NextText}Indexer.forward`` is decorated with ``@torch.no_grad()``
#     while owning three quantizable linears (``wq_b``, ``wk``, ``weights_proj``).
#     Wrapping them makes the same compiled quant function run under both grad-disabled
#     (indexer) and grad-enabled (rest of the block) modes. dynamo keys its cache by
#     *code object* -- shared by every WrapperLinear -- so the AOTAutograd backward then
#     fails with "cannot enter context: <Context ...> is already entered".
#   * transformers marks ``indexer.weights_proj`` in ``_keep_in_fp32_modules``, i.e. it
#     is not even supposed to leave FP32.
# The indexer is tiny anyway (``weights_proj`` outputs ``index_n_heads``), so there is
# no compression to gain here. ``get_glm_flash_ignore_layers`` additionally keeps the
# leading dense MLPs in full precision to work around a vLLM loading issue.
register_ignore_layers(
    matchers=[
        ModelTypeMatcher(r"glm_moe_dsa", mode="full"),
    ],
    ignore_layers=[get_glm_flash_ignore_layers, "weights_proj", "indexer"],  # get_glm_flash_ignore_layers: vllm issue
)

register_ignore_layers(
    matchers=[
        ModelTypeMatcher(r"glm5_next", mode="full"),
    ],
    ignore_layers=[
        get_glm_flash_ignore_layers,  # vllm issue
        "weights_proj",
        "indexer",
        "self_attn",
    ],
)

# step3p5
register_ignore_layers(
    matchers=[
        ModelTypeMatcher(r"step3p5", mode="full"),
    ],
    ignore_layers=[
        "g_proj",  # shape issue [96, 4096], 96 is not divisible by 64
        "moe.gate",
        "eh_proj",  # MTP layer
        "shared_head",  # MTP layer
        "layers.45",  # MTP layer, requiring g_idx in vLLM, skip it
    ],
)

register_ignore_layers(
    matchers=[
        ModelTypeMatcher(r"kimi_k25", mode="full"),
    ],
    ignore_layers=[
        "vision_tower",
        "mm_projector",
    ],
)

# qwen4: keep hyper_connection and MoE gate modules in full precision.
register_ignore_layers(
    matchers=[
        ArchitectureMatcher(r"Qwen4", mode="in"),
    ],
    ignore_layers=[
        "hyper_connection",
        "mlp.gate",  # MoE router gate
        "shared_expert",
    ],
)


def get_bagel_ignore_layers(model) -> list[str]:
    """Keep BAGEL generation-path modules in BF16.

    BAGEL uses `*_moe_gen` modules for the image-generation path. Quantizing
    them can reduce quality during the iterative denoising loop. Users can
    explicitly opt in with ``AR_QUANTIZE_BAGEL_MOE_GEN=1``.

    The standard Qwen2 attention and MLP projections are deliberately not
    ignored: they form BAGEL's normal text path and should follow AutoRound's
    usual transformer-layer policy.
    """
    from auto_round import envs

    ignore_layers = []
    if not envs.AR_QUANTIZE_BAGEL_MOE_GEN:
        ignore_layers.append("moe_gen")
    else:
        logger.warning(
            "AR_QUANTIZE_BAGEL_MOE_GEN is enabled. Quantizing BAGEL's image-generation experts may reduce image "
            "quality."
        )
    return ignore_layers


register_ignore_layers(
    matchers=[
        ModelTypeMatcher(r"bagel", mode="full"),
    ],
    ignore_layers=[
        get_bagel_ignore_layers,
    ],
)


# Architecture-specific layer_config defaults, keyed by ``config.model_type``.
# Nemotron-H Omni wrappers (Super VL, Bolt) keep their Mamba2 ``out_proj`` scales in
# BF16; every other model, including plain ``nemotron_h``, keeps the caller's scale_dtype.
_PRE_DEFINED_LAYER_CONFIG = {
    "nemotron_h_omni": {".*mixer.out_proj$": {"scale_dtype": torch.bfloat16}},
}


def get_predefined_layer_config(model: torch.nn.Module) -> dict:
    """Return predefined layer_config patterns for the model; user entries take precedence."""
    model_type = getattr(getattr(model, "config", None), "model_type", None)
    return _PRE_DEFINED_LAYER_CONFIG.get(model_type, {})


def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:
    layers = []
    for rule in _PRE_DEFINED_IGNORE_LAYERS:
        if all(m(model) for m in rule.matchers):
            for ignore_layer in rule.ignore_layers:
                if isinstance(ignore_layer, str):
                    layers.append(ignore_layer)
                else:
                    res = ignore_layer(model)
                    if isinstance(res, str):
                        layers.append(res)
                    else:
                        layers.extend(res)
            break
    config = getattr(model, "config", None)
    if not layers and is_moe_model_via_config(config):
        for name, _ in model.named_modules():
            if name.endswith((".gate", ".shared_expert_gate")):
                layers.append(name)

    return list(dict.fromkeys(layers))


# ---------------------------------------------------------------------------
# Architectures for which ``torch.compile`` is force-disabled during tuning.
# ---------------------------------------------------------------------------
@dataclass
class PreDefinedTorchCompileOff:
    """A rule that force-disables ``torch.compile`` for a family of models."""

    matchers: list[Callable[[Any], bool]]
    reason: str


_PRE_DEFINED_TORCH_COMPILE_OFF: list[PreDefinedTorchCompileOff] = []


def register_torch_compile_off(matchers: list[Callable[[Any], bool]], reason: str) -> None:
    """Register an architecture family for which ``torch.compile`` must stay off.

    Any matcher hitting is enough (``any``, not ``all``) so one rule can cover several
    equivalent identifiers (model_type and architecture name).
    """
    _PRE_DEFINED_TORCH_COMPILE_OFF.append(PreDefinedTorchCompileOff(matchers, reason))


def get_torch_compile_off_reason(model) -> str | None:
    """Return why ``torch.compile`` must stay off for this model, else ``None``.

    Accepts a model or anything exposing ``.config``.
    """
    if model is None:
        return None
    proxy = model if hasattr(model, "config") else SimpleNamespace(config=model)
    for rule in _PRE_DEFINED_TORCH_COMPILE_OFF:
        if any(matcher(proxy) for matcher in rule.matchers):
            return rule.reason
    return None


# DeepSeek V2/V3/V3.2/V4 and the GLM-5.3-Flash family (``glm_moe_dsa`` text /
# ``glm5_next`` multimodal) share a DSA indexer + fused MoE with shape-dependent
# control flow. On torch < 2.14.0 compiling the tuning graph there hits dynamo's
# recompile_limit and provides little benefit, so keep those runs in eager mode.
# From torch 2.14.0 onward these constraints are resolved, so we no longer force
# ``torch.compile`` off and instead honor the default / user-provided setting.
if not torch_version_at_least("2.14.0"):
    register_torch_compile_off(
        matchers=[
            ModelTypeMatcher("deepseek", mode="in"),
            ArchitectureMatcher("Deepseek", mode="in"),
            ArchitectureMatcher("DeepSeek", mode="in"),
        ],
        reason="the DeepSeek architecture is incompatible with torch.compile during tuning on torch < 2.14.0",
    )
    register_torch_compile_off(
        matchers=[
            ModelTypeMatcher(r"glm_moe_dsa", mode="full"),
            ModelTypeMatcher(r"glm5_next", mode="full"),
            ArchitectureMatcher(r"Glm5Next", mode="in"),
        ],
        reason="the GLM-5 architecture is incompatible with torch.compile during tuning on torch < 2.14.0",
    )


def _attach_gemma4_rotary_emb(model):
    """Attach ``_rotary_emb`` to each Gemma4 decoder layer.

    For transformers >= 5.6 the per-layer forward patch is unnecessary, but
    ``block_forward`` still needs access to ``rotary_emb`` (which lives on the
    parent ``Gemma4TextModel``) to recompute ``position_embeddings`` when the
    cached version from block 0 has the wrong dimension.
    """
    try:
        from transformers.models.gemma4.modeling_gemma4 import Gemma4TextModel
    except ImportError:
        return

    text_model = None
    for _, submodule in model.named_modules():
        if isinstance(submodule, Gemma4TextModel):
            text_model = submodule
            break

    if text_model is None:
        return

    # Create a single shared dict to propagate KV state between anchor/sharer layers.
    # Gemma4TextModel.forward in newer transformers uses the same pattern.
    shared_kv_states_global = {}

    for layer in text_model.layers:
        # Store in a plain list to prevent nn.Module from registering these
        # as child submodules (which would cause meta-tensor errors during .to(device)).
        object.__setattr__(layer, "_rotary_emb_ref", [text_model.rotary_emb])
        object.__setattr__(layer, "_shared_kv_states_global_ref", shared_kv_states_global)
        object.__setattr__(layer, "_autoround_special_replay", "gemma4")
        object.__setattr__(layer, "_gemma4_config_ref", text_model.config)


def _attach_gemma4_unified_rotary_emb(model):
    """Attach ``_rotary_emb`` to each Gemma4 decoder layer.

    For transformers >= 5.6 the per-layer forward patch is unnecessary, but
    ``block_forward`` still needs access to ``rotary_emb`` (which lives on the
    parent ``Gemma4TextModel``) to recompute ``position_embeddings`` when the
    cached version from block 0 has the wrong dimension.
    """
    try:
        from transformers.models.gemma4_unified import Gemma4UnifiedTextModel
    except ImportError:
        return

    text_model = None
    for _, submodule in model.named_modules():
        if isinstance(submodule, Gemma4UnifiedTextModel):
            text_model = submodule
            break

    if text_model is None:
        return

    # Create a single shared dict to propagate KV state between anchor/sharer layers.
    # Gemma4TextModel.forward in newer transformers uses the same pattern.
    shared_kv_states_global = {}

    for layer in text_model.layers:
        # Store in a plain list to prevent nn.Module from registering these
        # as child submodules (which would cause meta-tensor errors during .to(device)).
        object.__setattr__(layer, "_rotary_emb_ref", [text_model.rotary_emb])
        object.__setattr__(layer, "_shared_kv_states_global_ref", shared_kv_states_global)
        object.__setattr__(layer, "_autoround_special_replay", "gemma4")
        object.__setattr__(layer, "_gemma4_config_ref", text_model.config)


def load_next_step_diffusion(pretrained_model_name_or_path, device_str):
    try:
        from models.gen_pipeline import NextStepPipeline  # pylint: disable=E0401
    except ImportError:
        raise ImportError(
            "NextStepPipeline module not found. "
            + "Please navigate to the model file path and add it to your PYTHONPATH."
        )
    from transformers import AutoModel, AutoTokenizer

    tokenizer = AutoTokenizer.from_pretrained(
        pretrained_model_name_or_path, local_files_only=True, trust_remote_code=True
    )
    model = AutoModel.from_pretrained(pretrained_model_name_or_path, local_files_only=True, trust_remote_code=True)
    # The model is loaded onto the device because more than one block requires input data.
    pipe = NextStepPipeline(tokenizer=tokenizer, model=model).to(device=device_str, dtype=torch.bfloat16)

    def _nextstep_pipeline_fn(pipe, prompts, guidance_scale=7.5, num_inference_steps=28, generator=None, **kwargs):
        """Default pipeline_fn for NextStep models.

        Maps standard :class:`DiffusionCompressor` parameters to NextStep's
        ``generate_image`` API.  Pass a custom ``pipeline_fn`` to
        :class:`DiffusionCompressor` to override defaults or supply
        model-specific kwargs (e.g. ``hw``, ``positive_prompt``,
        ``cfg_schedule``, ``timesteps_shift``).
        """
        for prompt in (prompts if isinstance(prompts, list) else [prompts]):
            pipe.generate_image(
                prompt,
                cfg=guidance_scale,
                num_sampling_steps=num_inference_steps,
                **kwargs,
            )

    pipe._autoround_pipeline_fn = _nextstep_pipeline_fn
    return pipe, model


_PRE_DEFINED_FIXED_ATTR = {"gemma4_unified": {"has_variable_block_shape": True}}


def get_predefined_fixed_attr(model: torch.nn.Module) -> dict | None:
    """Return fixed compressor attributes for models that need special caching.

    For Gemma4 with transformers >= 5.6, each decoder block must cache its own
    inputs because sliding vs full-attention layers require different
    position_embeddings. Returns ``None`` for older transformers, which instead
    rely on the per-layer forward patch applied in ``_handle_special_model``.
    """
    import transformers
    from packaging import version

    config = getattr(model, "config", None)
    if config is None or not hasattr(config, "model_type"):
        return None
    attrs = _PRE_DEFINED_FIXED_ATTR.get(config.model_type)
    return attrs


# Cosmos3 world-model handler.
def _set_quantizable_bits(module, bits=4):
    """Mark all nn.Linear children as quantizable so AutoRound wraps them."""
    import torch.nn as nn

    for _n, _m in module.named_modules():
        if not isinstance(_m, nn.Linear):
            continue
        _m.bits = getattr(_m, "bits", bits)
        _m.group_size = getattr(_m, "group_size", 128)
        _m.sym = getattr(_m, "sym", True)
        _m.data_type = getattr(_m, "data_type", "int")
        _m.scale_dtype = getattr(_m, "scale_dtype", None)
        _m.act_bits = getattr(_m, "act_bits", 16)
        _m.act_sym = getattr(_m, "act_sym", None)
        _m.act_data_type = getattr(_m, "act_data_type", None)
        _m.act_dynamic = getattr(_m, "act_dynamic", True)


import contextlib
import threading

_cosmos3_forward_state = threading.local()


@contextlib.contextmanager
def _cosmos3_forward_mode():
    """Context flag marking a genuine `Cosmos3OmniPipeline.__call__` forward."""
    _cosmos3_forward_state.active = True
    try:
        yield
    finally:
        _cosmos3_forward_state.active = False


def _wrap_cosmos3_layer_dual_mode(layer):
    """Monkey-patch one `Cosmos3VLTextMoTDecoderLayer` in place."""
    if getattr(layer, "_autoround_dual_mode_patched", False):
        return
    orig_forward = layer.forward

    def _dual_mode_forward(self, und_seq, gen_seq, rotary_emb):
        if getattr(_cosmos3_forward_state, "active", False):
            return orig_forward(und_seq, gen_seq, rotary_emb)

        und_len = rotary_emb[0].shape[0]
        if und_seq.shape[0] != und_len:
            combined = und_seq
            und_seq = combined[:und_len]
            gen_seq = combined[und_len:]
        und_out, gen_out = orig_forward(und_seq, gen_seq, rotary_emb)
        return torch.cat([und_out, gen_out], dim=0)

    import types

    layer.forward = types.MethodType(_dual_mode_forward, layer)
    layer._autoround_dual_mode_patched = True


def _get_cosmos3_multimodal_block(model, quant_vision=False):
    """Get block names for Cosmos3."""
    if hasattr(model, "layers"):
        return [[f"layers.{i}" for i in range(len(model.layers))]]
    return []


def _bypass_cosmos3_safety_checker():
    """Patch Cosmos3 safety checker so calibration does not require cosmos_guardrail."""
    # Prefer an already-registered module.  Besides avoiding an unnecessary
    # import, this makes the optional dependency easy to substitute in tests
    # and in environments where diffusers lazily registers pipeline modules.
    module_name = "diffusers.pipelines.cosmos.pipeline_cosmos3_omni"
    pipeline_cosmos3_omni = sys.modules.get(module_name)
    if pipeline_cosmos3_omni is None:
        try:
            pipeline_cosmos3_omni = importlib.import_module(module_name)
        except ImportError:
            return

    safety_checker = getattr(pipeline_cosmos3_omni, "CosmosSafetyChecker", None)
    if safety_checker is None or getattr(safety_checker, "_autoround_patched", False):
        return

    def _patched_init(self, *args, **kwargs):
        torch.nn.Module.__init__(self)

    safety_checker.__init__ = _patched_init
    safety_checker.to = lambda self, *args, **kwargs: self
    safety_checker.check_text_safety = staticmethod(lambda prompt: True)
    safety_checker.check_video_safety = staticmethod(lambda frames: frames)
    safety_checker.dtype = property(lambda self: torch.float32)
    safety_checker.device = property(lambda self: torch.device("cpu"))
    safety_checker._autoround_patched = True


def _align_cosmos3_quant_config_to_vllm_omni(transformer_dir):
    """Rewrite the fused ``layers`` block name to vllm-omni's split module tree."""
    import json
    import os

    def _rewrite_blocks(blocks):
        if isinstance(blocks, str):
            blocks = [b.strip() for b in blocks.split(",") if b.strip()]
        if not isinstance(blocks, list):
            return blocks
        out = []
        for b in blocks:
            if b == "layers":
                out.extend(["language_model.layers", "gen_layers"])
            else:
                out.append(b)
        return out

    config_path = os.path.join(transformer_dir, "config.json")
    if not os.path.isfile(config_path):
        return
    with open(config_path, "r", encoding="utf-8") as f:
        config_data = json.load(f)
    qcfg = config_data.get("quantization_config")
    if not isinstance(qcfg, dict) or "block_name_to_quantize" not in qcfg:
        return
    rewritten = _rewrite_blocks(qcfg["block_name_to_quantize"])
    qcfg["block_name_to_quantize"] = rewritten
    with open(config_path, "w", encoding="utf-8") as f:
        json.dump(config_data, f, indent=2, sort_keys=True)

    quant_config_path = os.path.join(transformer_dir, "quantization_config.json")
    if os.path.isfile(quant_config_path):
        with open(quant_config_path, "r", encoding="utf-8") as f:
            quant_config_data = json.load(f)
        if "block_name_to_quantize" in quant_config_data:
            quant_config_data["block_name_to_quantize"] = rewritten
            with open(quant_config_path, "w", encoding="utf-8") as f:
                json.dump(quant_config_data, f, indent=2, sort_keys=True)


def load_cosmos3_diffusion(pretrained_model_name_or_path, device_str):
    """Load Cosmos3 from a diffusers checkpoint for AutoRound quantization.

    Returns a (pipe, model) pair where `model.layers` are the quantizable
    blocks.
    """
    import json
    import os
    import shutil

    from diffusers import Cosmos3OmniPipeline  # pylint: disable=import-error,no-name-in-module

    _bypass_cosmos3_safety_checker()

    diffusers_pipe = Cosmos3OmniPipeline.from_pretrained(pretrained_model_name_or_path, torch_dtype=torch.bfloat16)
    torch.set_grad_enabled(True)
    fused_model = diffusers_pipe.transformer

    for layer in fused_model.layers:
        _wrap_cosmos3_layer_dual_mode(layer)
    _set_quantizable_bits(fused_model.layers)

    # Read transformer config for model dimensions / metadata only.
    config_path = os.path.join(pretrained_model_name_or_path, "transformer", "config.json")
    with open(config_path, "r") as f:
        cfg = json.load(f)

    import torch.nn as nn
    from transformers import PretrainedConfig

    class _Cosmos3VllmOmniConfig(PretrainedConfig):
        model_type = "cosmos3_vllm_omni"

        def __init__(self, **kwargs):
            kwargs.pop("model_type", None)
            super().__init__(**kwargs)

    class Cosmos3VllmOmniQuantModel(nn.Module):
        def __init__(self):
            super().__init__()
            self.config = _Cosmos3VllmOmniConfig(**cfg)
            self.config._class_name = "Cosmos3VFMTransformer"
            self.config.architectures = ["Cosmos3VFMTransformer"]
            self.config.name_or_path = pretrained_model_name_or_path
            self.layers = fused_model.layers

        @property
        def dtype(self):
            try:
                return next(self.parameters()).dtype
            except StopIteration:
                return torch.float32

        @property
        def device(self):
            try:
                return next(self.parameters()).device
            except StopIteration:
                return torch.device("cpu")

        def save_pretrained(self, save_directory, max_shard_size="5GB", safe_serialization=True, **kwargs):
            os.makedirs(save_directory, exist_ok=True)
            self.config.save_pretrained(save_directory)
            state_dict = {name: tensor.detach().cpu() for name, tensor in self.state_dict().items()}
            if safe_serialization:
                from safetensors.torch import save_file

                save_file(state_dict, os.path.join(save_directory, "model.safetensors"))
            else:
                torch.save(state_dict, os.path.join(save_directory, "pytorch_model.bin"))

    model = Cosmos3VllmOmniQuantModel()
    logger.info("Cosmos3: quantizing %d decoder-block layers", len(fused_model.layers))

    class _Cosmos3Pipe:
        _autoround_diffusion_pipe = True

        def __init__(self, transformer, diffusers_pipe, pipe_cfg):
            self.transformer = transformer
            self.model = transformer
            self.diffusers_pipe = diffusers_pipe
            self.config = pipe_cfg
            self.components = {"transformer": transformer}
            self.dtype = torch.bfloat16

        @property
        def device(self):
            return next(self.transformer.parameters()).device

        def to(self, *args, **kwargs):
            self.diffusers_pipe.to(*args, **kwargs)
            for component in vars(self.diffusers_pipe).values():
                keep_fp32 = getattr(component, "_keep_in_fp32_modules", None)
                if not isinstance(component, torch.nn.Module) or not keep_fp32:
                    continue
                for sub_name, sub_module in component.named_modules():
                    if any(sub_name == name or sub_name.startswith(name + ".") for name in keep_fp32):
                        sub_module.to(torch.float32)
            return self

        def save_config(self, save_directory):
            os.makedirs(save_directory, exist_ok=True)
            model_index_path = os.path.join(pretrained_model_name_or_path, "model_index.json")
            with open(model_index_path, "r", encoding="utf-8") as f:
                model_index = json.load(f)

            model_index["_class_name"] = "Cosmos3OmniDiffusersPipeline"
            model_index["_name_or_path"] = pretrained_model_name_or_path
            for component_name, component_spec in model_index.items():
                if (
                    component_name.startswith("_")
                    or component_name == "transformer"
                    or not isinstance(component_spec, list)
                ):
                    continue
                source = os.path.join(pretrained_model_name_or_path, component_name)
                target = os.path.join(save_directory, component_name)
                if os.path.isdir(source):
                    shutil.copytree(source, target, dirs_exist_ok=True)
            with open(os.path.join(save_directory, "model_index.json"), "w", encoding="utf-8") as f:
                f.write(json.dumps(model_index, indent=2, sort_keys=True) + "\n")

            _SKIP_TOPLEVEL = {
                "model_index.json",
                "checkpoint.json",
                "model.safetensors.index.json",
            }
            for entry in os.listdir(pretrained_model_name_or_path):
                src = os.path.join(pretrained_model_name_or_path, entry)
                if not os.path.isfile(src):
                    continue
                if entry in _SKIP_TOPLEVEL:
                    continue
                if entry.endswith((".json", ".txt", ".model")):
                    shutil.copy2(src, os.path.join(save_directory, entry))

            # Align the exported quant metadata with vllm-omni's runtime module tree.
            _align_cosmos3_quant_config_to_vllm_omni(os.path.join(save_directory, "transformer"))

        def __call__(self, prompts=None, guidance_scale=6.0, num_inference_steps=35, generator=None, **kwargs):
            """Calibration forward: drives the actual Cosmos3OmniPipeline."""
            prompt_list = prompts if isinstance(prompts, list) else [prompts]
            with _cosmos3_forward_mode():
                for prompt in prompt_list:
                    self.diffusers_pipe(
                        prompt=prompt,
                        num_frames=1,
                        num_inference_steps=num_inference_steps,
                        guidance_scale=guidance_scale,
                        generator=generator,
                        output_type="latent",
                        enable_safety_check=False,
                    )

    pipe = _Cosmos3Pipe(model, diffusers_pipe, dict(diffusers_pipe.config))

    return pipe, model


# Register Cosmos3 block detection.
SPECIAL_MULTIMODAL_BLOCK["cosmos3_vllm_omni"] = _get_cosmos3_multimodal_block
SPECIAL_SHARED_CACHE_KEYS["Cosmos3VllmOmniQuantModel"] = ("rotary_emb",)