PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support llama4 quant (#744)
Wang, Mengni2df63f27dadb
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 2c30b704f..c4915f779 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -30,6 +30,7 @@ from auto_round.data_type import QUANT_FUNC_WITH_DTYPEfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_sizefrom auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelTypefrom auto_round.low_cpu_mem.utils import get_layers_before_block+from auto_round.special_model_handler import _handle_moe_modelfrom auto_round.utils import (INNER_SUPPORTED_LAYER_TYPES,SUPPORTED_DTYPES,@@ -222,6 +223,7 @@ class AutoRound(object):"AutoRound does not support parameters on meta device. ""Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU.")+ model = _handle_moe_model(model)# self.model_orig_dtype = model.dtypeself.device = detect_device(device) ##must place after llm_load_model, because this one will convert autodiff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 95ba736bd..67b64e113 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -27,6 +27,7 @@ from auto_round.special_model_handler import (from ..autoround import AutoRoundfrom ..low_cpu_mem.utils import get_layers_before_blockfrom ..utils import (+ check_to_quantized,clear_memory,detect_device,extract_block_names_to_str,@@ -460,6 +461,6 @@ class AutoRoundMLLM(AutoRound):for layer_name in layer_config.keys():for vlm_key in VISUAL_KEYS:- if vlm_key in layer_name:+ if vlm_key in layer_name and check_to_quantized(layer_config[layer_name]):return Truereturn quant_nontext_modulediff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 3e25a32d3..cf0bb90a5 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -36,6 +36,72 @@ SPECIAL_SHARED_CACHE_KEYS = {}SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)+CONVERT_EXPERT_TO_LINEAR_MODELS = ["llama4"]+++def _get_moe_converter(config):+ import torch+ from transformers.modeling_utils import no_init_weights++ # https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/llama4.py+ if config.model_type == "llama4":+ from transformers.models.llama4.modeling_llama4 import Llama4TextMLP++ class SequentialLlama4TextExperts(torch.nn.ModuleList):+ def __init__(self, config, original):+ self.num_experts = original.gate_up_proj.shape[0]+ with no_init_weights():+ super().__init__([Llama4TextMLP(config) for _ in range(self.num_experts)])+ intermediate_size = original.down_proj.shape[1]++ for i in range(self.num_experts):+ gate_up = original.gate_up_proj[i]+ down = original.down_proj[i]+ gate_proj = gate_up[:, :intermediate_size]+ up_proj = gate_up[:, intermediate_size:]++ self[i].gate_proj.weight.data = gate_proj.t().contiguous()+ self[i].up_proj.weight.data = up_proj.t().contiguous()+ self[i].down_proj.weight.data = down.t().contiguous()++ class SequentialLlama4TextMoe(torch.nn.Module):+ def __init__(self, config, original):+ super().__init__()+ self.top_k = config.num_experts_per_tok+ self.hidden_dim = config.hidden_size+ self.num_experts = config.num_local_experts+ self.experts = SequentialLlama4TextExperts(config, original.experts)+ self.router = original.router+ self.shared_expert = original.shared_expert++ def forward(self, hidden_states: torch.Tensor):+ hidden_states = hidden_states.reshape(-1, self.hidden_dim)+ router_logits = self.router(hidden_states)+ if isinstance(router_logits, tuple):+ router_scores, router_logits = router_logits+ router_scores = router_scores.t()+ else:+ # transformers < 4.54.0 only returns router_logits+ router_top_value, router_indices = torch.topk(router_logits, self.top_k, dim=1)++ router_scores = (+ torch.full_like(router_logits, float("-inf"))+ .scatter_(1, router_indices, router_top_value)+ .transpose(0, 1)+ )+ router_scores = torch.sigmoid(router_scores.float()).to(hidden_states.dtype)++ out = self.shared_expert(hidden_states)+ for i in range(self.num_experts):+ out += self.experts[i](hidden_states) * router_scores[i].reshape(-1, 1)++ return out, router_logits++ return SequentialLlama4TextMoe, config.get_text_config(), "Llama4TextMoe"++ else:+ raise ValueError(f"Currently moe converter only supports llama4 model_type, but get {config.model_type}")+def _handle_special_model(model):if model.config.model_type == "deepseek_vl_v2":@@ -45,6 +111,28 @@ def _handle_special_model(model):return model+def _handle_moe_model(model):+ if model.config.model_type in CONVERT_EXPERT_TO_LINEAR_MODELS:+ from tqdm import tqdm++ from auto_round.utils import clear_memory++ new_moe_class, convert_config, orig_cls_name = _get_moe_converter(model.config)+ model = model.to("cpu")+ clear_memory()++ for name, module in tqdm(model.named_modules(), desc="Converting model"):+ cls_name = module.__class__.__name__+ if cls_name == orig_cls_name:+ new_module = new_moe_class(config=convert_config, original=module)+ parent, child = name.rsplit(".", maxsplit=1)+ parent = model.get_submodule(parent)+ setattr(parent, child, new_module)++ print("warning, Llama4 experts are converted, the quantized model can not run on transformers.")+ return model++def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):model.forward = model.language.forwardblock_names = []