PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add support for MiMo-V2-Flash (#1718)
Heng Guoe62d29d1dac1
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex 03a98dc12..d49999225 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -200,6 +200,91 @@ def _patch_tensor_get_dtype_for_prequantized_loading():torch.Tensor.get_dtype = _tensor_get_dtype+def _patch_default_rope_init():+ """Restore legacy ``rope_type='default'`` support for older remote-code models.++ Some third-party model implementations still look up ``ROPE_INIT_FUNCTIONS["default"]``+ directly when no explicit rope scaling is configured. Newer transformers releases no+ longer register that key, even though the standard RoPE parameterization is still valid.+ Register a small compatibility shim that computes the original default inverse frequencies.+ """+ try:+ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS+ except ImportError:+ return++ if "default" in ROPE_INIT_FUNCTIONS:+ return++ def _compute_default_rope_parameters(config=None, device=None, seq_len=None, layer_type=None):+ del seq_len # Unused for the default RoPE type.++ if config is None:+ raise ValueError("config must be provided to compute default RoPE parameters")++ rope_parameters = {}+ if hasattr(config, "standardize_rope_params") and hasattr(config, "rope_parameters"):+ config.standardize_rope_params()+ rope_parameters = config.rope_parameters[layer_type] if layer_type is not None else config.rope_parameters+ elif hasattr(config, "rope_scaling") and isinstance(config.rope_scaling, dict):+ rope_parameters = config.rope_scaling++ base = rope_parameters.get("rope_theta", getattr(config, "rope_theta", 10000.0))+ partial_rotary_factor = rope_parameters.get(+ "partial_rotary_factor", getattr(config, "partial_rotary_factor", 1.0)+ )+ head_dim = getattr(config, "head_dim", None) or config.hidden_size // config.num_attention_heads+ dim = int(head_dim * partial_rotary_factor)+ if dim <= 0:+ raise ValueError(f"Invalid RoPE dimension {dim} for config {config.__class__.__name__}")++ inv_freq = 1.0 / (+ base ** (torch.arange(0, dim, 2, dtype=torch.int64).to(device=device, dtype=torch.float) / dim)+ )+ return inv_freq, 1.0++ ROPE_INIT_FUNCTIONS["default"] = _compute_default_rope_parameters+++def _patch_rotary_embedding_init_for_legacy_remote_code():+ """Patch transformers weight init to support legacy remote-code RotaryEmbedding modules.++ transformers 5.5+ expects RotaryEmbedding modules using ``rope_type='default'``+ to expose ``compute_default_rope_parameters``. Older remote-code models often do+ not define that helper and instead looked up the default entry in+ ``ROPE_INIT_FUNCTIONS`` directly.+ """+ pre_trained_model = transformers.modeling_utils.PreTrainedModel+ original_init_weights = pre_trained_model._init_weights++ if getattr(original_init_weights, "_auto_round_rotary_default_patch", False):+ return++ @wraps(original_init_weights)+ def _patched_init_weights(self, module):+ try:+ return original_init_weights(self, module)+ except AttributeError as exc:+ if (+ "compute_default_rope_parameters" not in str(exc)+ or "RotaryEmbedding" not in module.__class__.__name__+ or not hasattr(module, "original_inv_freq")+ or not hasattr(module, "inv_freq")+ or getattr(module, "rope_type", None) != "default"+ ):+ raise++ from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS++ buffer_value, _ = ROPE_INIT_FUNCTIONS["default"](module.config)+ with torch.no_grad():+ module.inv_freq.copy_(buffer_value)+ module.original_inv_freq.copy_(buffer_value)++ _patched_init_weights._auto_round_rotary_default_patch = True+ pre_trained_model._init_weights = _patched_init_weights++# TODO: only AutoModelForCausalLM is patched; other Auto* classes are not covered yetdef monkey_patch_transformers():transformers_version = getattr(transformers, "__version__", None)@@ -228,6 +313,8 @@ def monkey_patch_transformers():# transformers 5.3.0 calls tensor.get_dtype() on plain torch.Tensor objects# while loading pre-quantized checkpoints._patch_tensor_get_dtype_for_prequantized_loading()+ _patch_default_rope_init()+ _patch_rotary_embedding_init_for_legacy_remote_code()if parsed_version >= version.parse("4.56.0"):_patch_classmethod_kwargs(transformers.AutoModelForCausalLM, "from_pretrained", torch_dtype="dtype")else:@@ -246,6 +333,7 @@ def monkey_patch_model(model) -> None:class-level patches in ``monkey_patch_transformers``)."""_patch_prepare_inputs_for_generation(model)+ _patch_mimo_attention_forward(model)def _patch_prepare_inputs_for_generation(model) -> None:@@ -269,6 +357,54 @@ def _patch_prepare_inputs_for_generation(model) -> None:_patch_qwen3_omni_moe_talker(model)+def _patch_mimo_attention_forward(model) -> None:+ """Patch MiMo remote-code attention helpers for newer transformers call sites."""+ # Check if this is a MiMo model by class name (remote code models may not have model_type)+ model_class_name = model.__class__.__name__+ logger.info(f"_patch_mimo_attention_forward called for {model_class_name}")++ if "MiMo" not in model_class_name and "mimo" not in model_class_name.lower():+ logger.info(f"Skipping patch: not a MiMo model (class name: {model_class_name})")+ return++ try:+ module = importlib.import_module(model.__class__.__module__)+ logger.info(f"Imported module: {model.__class__.__module__}")+ except ImportError as e:+ logger.warning(f"Could not import module {model.__class__.__module__}: {e}")+ return++ eager_attention_forward = getattr(module, "eager_attention_forward", None)+ logger.info(f"eager_attention_forward found: {eager_attention_forward is not None}")++ if eager_attention_forward is None:+ logger.info("Skipping patch: eager_attention_forward not found in module")+ return++ if getattr(eager_attention_forward, "_auto_round_mimo_patch", False):+ logger.info("Skipping patch: already patched")+ return++ @wraps(eager_attention_forward)+ def _patched_eager_attention_forward(+ module_obj,+ query,+ key,+ value,+ attention_mask,+ scaling,+ dropout=0.0,+ sinks=None,+ **kwargs,+ ):+ del kwargs+ return eager_attention_forward(module_obj, query, key, value, attention_mask, scaling, dropout, sinks)++ _patched_eager_attention_forward._auto_round_mimo_patch = True+ module.eager_attention_forward = _patched_eager_attention_forward+ logger.info(f"Successfully patched eager_attention_forward in {model.__class__.__module__}")++def _patch_qwen25_omni_talker(model) -> None:"""Patch Qwen2.5-Omni talker prepare_inputs_for_generation."""talker = getattr(model, "talker", None)diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 3f67deb63..210905e84 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -29,6 +29,7 @@ from auto_round import envsfrom auto_round.export.export_to_gguf.config import ModelTypefrom auto_round.logger import loggerfrom auto_round.schemes import QuantizationScheme+from auto_round.utils.common import monkey_patch_modelfrom auto_round.utils.weight_handler import (_dequant_fp8_linear_weight,check_and_mark_quantized_module,@@ -363,6 +364,7 @@ def llm_load_model()model = model.eval()+ monkey_patch_model(model)check_and_mark_quantized_module(model)handle_generation_config(model)model = _to_model_dtype(model, model_dtype)diff --git a/auto_round/utils/weight_handler.py b/auto_round/utils/weight_handler.pyindex 6cd3e07da..b58a030c7 100755--- a/auto_round/utils/weight_handler.py+++ b/auto_round/utils/weight_handler.py@@ -451,8 +451,25 @@ def _pad_block_fp8_weight_naive(weight, orig_M, orig_N = _pad_weight(weight, block_size)M, N = weight.shape[-2:]- assert weight_scale_m == M // block_size_m- assert weight_scale_n == N // block_size_n+ # Check if scale tensor is large enough to cover the weight dimensions+ # Some models may use over-provisioned scale tensors (e.g., for memory alignment),+ # but undersized scales are invalid and should be rejected early+ expected_scale_m = M // block_size_m+ expected_scale_n = N // block_size_n+ if weight_scale_m < expected_scale_m or weight_scale_n < expected_scale_n:+ raise ValueError(+ "FP8 weight scale shape is smaller than required for the padded weight: "+ f"weight_shape={tuple(weight.shape)}, scale_shape={tuple(weight_scale.shape)}, block_size={block_size}"+ )++ # Handle over-provisioned scale tensors: if the scale tensor covers more blocks+ # than the weight dimensions require, pad the weight to match the scale coverage.+ target_M = max(M, weight_scale_m * block_size_m)+ target_N = max(N, weight_scale_n * block_size_n)+ if target_M != M or target_N != N:+ pad_M = target_M - M+ pad_N = target_N - N+ weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)return weight, orig_M, orig_N