Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support llama4 quant (#744)

Wang, Mengni2df63f27dadb
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 2c30b704f..c4915f779 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -30,6 +30,7 @@ from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
from auto_round.export.export_to_gguf.config import GGUF_CONFIG, GGUF_INNER_CONFIG, ModelType
from auto_round.low_cpu_mem.utils import get_layers_before_block
+from auto_round.special_model_handler import _handle_moe_model
from auto_round.utils import (
INNER_SUPPORTED_LAYER_TYPES,
SUPPORTED_DTYPES,
@@ -222,6 +223,7 @@ class AutoRound(object):
"AutoRound does not support parameters on meta device. "
"Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."
)
+ model = _handle_moe_model(model)
# self.model_orig_dtype = model.dtype
self.device = detect_device(device) ##must place after llm_load_model, because this one will convert auto
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 95ba736bd..67b64e113 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -27,6 +27,7 @@ from auto_round.special_model_handler import (
from ..autoround import AutoRound
from ..low_cpu_mem.utils import get_layers_before_block
from ..utils import (
+ check_to_quantized,
clear_memory,
detect_device,
extract_block_names_to_str,
@@ -460,6 +461,6 @@ class AutoRoundMLLM(AutoRound):
for layer_name in layer_config.keys():
for vlm_key in VISUAL_KEYS:
- if vlm_key in layer_name:
+ if vlm_key in layer_name and check_to_quantized(layer_config[layer_name]):
return True
return quant_nontext_module
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 3e25a32d3..cf0bb90a5 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -36,6 +36,72 @@ SPECIAL_SHARED_CACHE_KEYS = {
}
SPECIAL_SHARED_CACHE_KEYS["MiniMaxText01ForCausalLM"] = ("slope_rate",)
+CONVERT_EXPERT_TO_LINEAR_MODELS = ["llama4"]
+
+
+def _get_moe_converter(config):
+ import torch
+ from transformers.modeling_utils import no_init_weights
+
+ # https://github.com/vllm-project/llm-compressor/blob/main/src/llmcompressor/modeling/llama4.py
+ if config.model_type == "llama4":
+ from transformers.models.llama4.modeling_llama4 import Llama4TextMLP
+
+ class SequentialLlama4TextExperts(torch.nn.ModuleList):
+ def __init__(self, config, original):
+ self.num_experts = original.gate_up_proj.shape[0]
+ with no_init_weights():
+ super().__init__([Llama4TextMLP(config) for _ in range(self.num_experts)])
+ intermediate_size = original.down_proj.shape[1]
+
+ for i in range(self.num_experts):
+ gate_up = original.gate_up_proj[i]
+ down = original.down_proj[i]
+ gate_proj = gate_up[:, :intermediate_size]
+ up_proj = gate_up[:, intermediate_size:]
+
+ self[i].gate_proj.weight.data = gate_proj.t().contiguous()
+ self[i].up_proj.weight.data = up_proj.t().contiguous()
+ self[i].down_proj.weight.data = down.t().contiguous()
+
+ class SequentialLlama4TextMoe(torch.nn.Module):
+ def __init__(self, config, original):
+ super().__init__()
+ self.top_k = config.num_experts_per_tok
+ self.hidden_dim = config.hidden_size
+ self.num_experts = config.num_local_experts
+ self.experts = SequentialLlama4TextExperts(config, original.experts)
+ self.router = original.router
+ self.shared_expert = original.shared_expert
+
+ def forward(self, hidden_states: torch.Tensor):
+ hidden_states = hidden_states.reshape(-1, self.hidden_dim)
+ router_logits = self.router(hidden_states)
+ if isinstance(router_logits, tuple):
+ router_scores, router_logits = router_logits
+ router_scores = router_scores.t()
+ else:
+ # transformers < 4.54.0 only returns router_logits
+ router_top_value, router_indices = torch.topk(router_logits, self.top_k, dim=1)
+
+ router_scores = (
+ torch.full_like(router_logits, float("-inf"))
+ .scatter_(1, router_indices, router_top_value)
+ .transpose(0, 1)
+ )
+ router_scores = torch.sigmoid(router_scores.float()).to(hidden_states.dtype)
+
+ out = self.shared_expert(hidden_states)
+ for i in range(self.num_experts):
+ out += self.experts[i](hidden_states) * router_scores[i].reshape(-1, 1)
+
+ return out, router_logits
+
+ return SequentialLlama4TextMoe, config.get_text_config(), "Llama4TextMoe"
+
+ else:
+ raise ValueError(f"Currently moe converter only supports llama4 model_type, but get {config.model_type}")
+
def _handle_special_model(model):
if model.config.model_type == "deepseek_vl_v2":
@@ -45,6 +111,28 @@ def _handle_special_model(model):
return model
+def _handle_moe_model(model):
+ if model.config.model_type in CONVERT_EXPERT_TO_LINEAR_MODELS:
+ from tqdm import tqdm
+
+ from auto_round.utils import clear_memory
+
+ new_moe_class, convert_config, orig_cls_name = _get_moe_converter(model.config)
+ model = model.to("cpu")
+ clear_memory()
+
+ for name, module in tqdm(model.named_modules(), desc="Converting model"):
+ cls_name = module.__class__.__name__
+ if cls_name == orig_cls_name:
+ new_module = new_moe_class(config=convert_config, original=module)
+ parent, child = name.rsplit(".", maxsplit=1)
+ parent = model.get_submodule(parent)
+ setattr(parent, child, new_module)
+
+ print("warning, Llama4 experts are converted, the quantized model can not run on transformers.")
+ return model
+
+
def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):
model.forward = model.language.forward
block_names = []