Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add support for MiMo-V2-Flash (#1718)

Heng Guoe62d29d1dac1
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index 03a98dc12..d49999225 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -200,6 +200,91 @@ def _patch_tensor_get_dtype_for_prequantized_loading():
torch.Tensor.get_dtype = _tensor_get_dtype
+def _patch_default_rope_init():
+ """Restore legacy ``rope_type='default'`` support for older remote-code models.
+
+ Some third-party model implementations still look up ``ROPE_INIT_FUNCTIONS["default"]``
+ directly when no explicit rope scaling is configured. Newer transformers releases no
+ longer register that key, even though the standard RoPE parameterization is still valid.
+ Register a small compatibility shim that computes the original default inverse frequencies.
+ """
+ try:
+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
+ except ImportError:
+ return
+
+ if "default" in ROPE_INIT_FUNCTIONS:
+ return
+
+ def _compute_default_rope_parameters(config=None, device=None, seq_len=None, layer_type=None):
+ del seq_len # Unused for the default RoPE type.
+
+ if config is None:
+ raise ValueError("config must be provided to compute default RoPE parameters")
+
+ rope_parameters = {}
+ if hasattr(config, "standardize_rope_params") and hasattr(config, "rope_parameters"):
+ config.standardize_rope_params()
+ rope_parameters = config.rope_parameters[layer_type] if layer_type is not None else config.rope_parameters
+ elif hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict):
+ rope_parameters = config.rope_scaling
+
+ base = rope_parameters.get("rope_theta", getattr(config, "rope_theta", 10000.0))
+ partial_rotary_factor = rope_parameters.get(
+ "partial_rotary_factor", getattr(config, "partial_rotary_factor", 1.0)
+ )
+ head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads
+ dim = int(head_dim * partial_rotary_factor)
+ if dim <= 0:
+ raise ValueError(f"Invalid RoPE dimension {dim} for config {config.__class__.__name__}")
+
+ inv_freq = 1.0 / (
+ base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim)
+ )
+ return inv_freq, 1.0
+
+ ROPE_INIT_FUNCTIONS["default"] = _compute_default_rope_parameters
+
+
+def _patch_rotary_embedding_init_for_legacy_remote_code():
+ """Patch transformers weight init to support legacy remote-code RotaryEmbedding modules.
+
+ transformers 5.5+ expects RotaryEmbedding modules using ``rope_type='default'``
+ to expose ``compute_default_rope_parameters``. Older remote-code models often do
+ not define that helper and instead looked up the default entry in
+ ``ROPE_INIT_FUNCTIONS`` directly.
+ """
+ pre_trained_model = transformers.modeling_utils.PreTrainedModel
+ original_init_weights = pre_trained_model._init_weights
+
+ if getattr(original_init_weights, "_auto_round_rotary_default_patch", False):
+ return
+
+ @wraps(original_init_weights)
+ def _patched_init_weights(self, module):
+ try:
+ return original_init_weights(self, module)
+ except AttributeError as exc:
+ if (
+ "compute_default_rope_parameters" not in str(exc)
+ or "RotaryEmbedding" not in module.__class__.__name__
+ or not hasattr(module, "original_inv_freq")
+ or not hasattr(module, "inv_freq")
+ or getattr(module, "rope_type", None) != "default"
+ ):
+ raise
+
+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
+
+ buffer_value, _ = ROPE_INIT_FUNCTIONS["default"](module.config)
+ with torch.no_grad():
+ module.inv_freq.copy_(buffer_value)
+ module.original_inv_freq.copy_(buffer_value)
+
+ _patched_init_weights._auto_round_rotary_default_patch = True
+ pre_trained_model._init_weights = _patched_init_weights
+
+
# TODO: only AutoModelForCausalLM is patched; other Auto* classes are not covered yet
def monkey_patch_transformers():
transformers_version = getattr(transformers, "__version__", None)
@@ -228,6 +313,8 @@ def monkey_patch_transformers():
# transformers 5.3.0 calls tensor.get_dtype() on plain torch.Tensor objects
# while loading pre-quantized checkpoints.
_patch_tensor_get_dtype_for_prequantized_loading()
+ _patch_default_rope_init()
+ _patch_rotary_embedding_init_for_legacy_remote_code()
if parsed_version >= version.parse("4.56.0"):
_patch_classmethod_kwargs(transformers.AutoModelForCausalLM, "from_pretrained", torch_dtype="dtype")
else:
@@ -246,6 +333,7 @@ def monkey_patch_model(model) -> None:
class-level patches in ``monkey_patch_transformers``).
"""
_patch_prepare_inputs_for_generation(model)
+ _patch_mimo_attention_forward(model)
def _patch_prepare_inputs_for_generation(model) -> None:
@@ -269,6 +357,54 @@ def _patch_prepare_inputs_for_generation(model) -> None:
_patch_qwen3_omni_moe_talker(model)
+def _patch_mimo_attention_forward(model) -> None:
+ """Patch MiMo remote-code attention helpers for newer transformers call sites."""
+ # Check if this is a MiMo model by class name (remote code models may not have model_type)
+ model_class_name = model.__class__.__name__
+ logger.info(f"_patch_mimo_attention_forward called for {model_class_name}")
+
+ if "MiMo" not in model_class_name and "mimo" not in model_class_name.lower():
+ logger.info(f"Skipping patch: not a MiMo model (class name: {model_class_name})")
+ return
+
+ try:
+ module = importlib.import_module(model.__class__.__module__)
+ logger.info(f"Imported module: {model.__class__.__module__}")
+ except ImportError as e:
+ logger.warning(f"Could not import module {model.__class__.__module__}: {e}")
+ return
+
+ eager_attention_forward = getattr(module, "eager_attention_forward", None)
+ logger.info(f"eager_attention_forward found: {eager_attention_forward is not None}")
+
+ if eager_attention_forward is None:
+ logger.info("Skipping patch: eager_attention_forward not found in module")
+ return
+
+ if getattr(eager_attention_forward, "_auto_round_mimo_patch", False):
+ logger.info("Skipping patch: already patched")
+ return
+
+ @wraps(eager_attention_forward)
+ def _patched_eager_attention_forward(
+ module_obj,
+ query,
+ key,
+ value,
+ attention_mask,
+ scaling,
+ dropout=0.0,
+ sinks=None,
+ **kwargs,
+ ):
+ del kwargs
+ return eager_attention_forward(module_obj, query, key, value, attention_mask, scaling, dropout, sinks)
+
+ _patched_eager_attention_forward._auto_round_mimo_patch = True
+ module.eager_attention_forward = _patched_eager_attention_forward
+ logger.info(f"Successfully patched eager_attention_forward in {model.__class__.__module__}")
+
+
def _patch_qwen25_omni_talker(model) -> None:
"""Patch Qwen2.5-Omni talker prepare_inputs_for_generation."""
talker = getattr(model, "talker", None)
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 3f67deb63..210905e84 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -29,6 +29,7 @@ from auto_round import envs
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.logger import logger
from auto_round.schemes import QuantizationScheme
+from auto_round.utils.common import monkey_patch_model
from auto_round.utils.weight_handler import (
_dequant_fp8_linear_weight,
check_and_mark_quantized_module,
@@ -363,6 +364,7 @@ def llm_load_model(
)
model = model.eval()
+ monkey_patch_model(model)
check_and_mark_quantized_module(model)
handle_generation_config(model)
model = _to_model_dtype(model, model_dtype)
diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.py
index 6cd3e07da..b58a030c7 100755
--- a/auto_round/utils/weight_handler.py
+++ b/auto_round/utils/weight_handler.py
@@ -451,8 +451,25 @@ def _pad_block_fp8_weight_naive(
weight, orig_M, orig_N = _pad_weight(weight, block_size)
M, N = weight.shape[-2:]
- assert weight_scale_m == M // block_size_m
- assert weight_scale_n == N // block_size_n
+ # Check if scale tensor is large enough to cover the weight dimensions
+ # Some models may use over-provisioned scale tensors (e.g., for memory alignment),
+ # but undersized scales are invalid and should be rejected early
+ expected_scale_m = M // block_size_m
+ expected_scale_n = N // block_size_n
+ if weight_scale_m < expected_scale_m or weight_scale_n < expected_scale_n:
+ raise ValueError(
+ "FP8 weight scale shape is smaller than required for the padded weight: "
+ f"weight_shape={tuple(weight.shape)}, scale_shape={tuple(weight_scale.shape)}, block_size={block_size}"
+ )
+
+ # Handle over-provisioned scale tensors: if the scale tensor covers more blocks
+ # than the weight dimensions require, pad the weight to match the scale coverage.
+ target_M = max(M, weight_scale_m * block_size_m)
+ target_N = max(N, weight_scale_n * block_size_n)
+ if target_M != M or target_N != N:
+ pad_M = target_M - M
+ pad_N = target_N - N
+ weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)
return weight, orig_M, orig_N