PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine global scale calculation to blockwise (#1421)
Weiweia554a107518d
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 65b4b9fc1..e855fd0ce 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -52,7 +52,7 @@ from auto_round.compressors.utils import (set_layer_config,)from auto_round.data_type import QUANT_FUNC_WITH_DTYPE-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, update_block_global_scale_if_neededfrom auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIGfrom auto_round.formats import OutputFormat, get_formatsfrom auto_round.logger import logger@@ -1274,27 +1274,6 @@ class BaseCompressor(object):all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]self.all_to_quantized_module_names = all_to_quantized_module_names- if is_nv_fp(self.data_type):- # FIXME: (yiliu30) change it to block-wise after we refactor the quantization code and- # https://github.com/intel/auto-round/issues/1331- materialize_model_(self.model)- self.model.to("cpu")- from auto_round.data_type.nvfp import calculate_gparam- from auto_round.data_type.utils import update_fused_layer_global_scales-- pbar = tqdm(all_to_quantized_module_names)- for name in pbar:- pbar.set_description(f"Calculate weight global scale: {name}")- m = get_module(self.model, name)- m = convert_module_to_hp_if_necessary(m, self.amp_dtype, self.device)- set_module(self.model, name, m)- weight_global_scale = calculate_gparam(m.weight, self.group_size)- setattr(m, "weight_global_scale", weight_global_scale)-- logger.info("Start to update fused layer global scales, it may take some time.")- for name, module in self.model.named_modules():- update_fused_layer_global_scales(module)- logger.info("Finished updating fused layer global scales.")if not (any(fmt.is_gguf() for fmt in getattr(self, "formats", [])) or self.super_bits is not None):self._quantize_embedding_layer() # leave to gguf itself to handle@@ -1485,7 +1464,9 @@ class BaseCompressor(object):block = get_module(self.model, block_name)materialize_model_(block)block.to("cpu")- convert_module_to_hp_if_necessary(block, dtype=self.amp_dtype, device=self.device)++ block = convert_module_to_hp_if_necessary(block, dtype=self.amp_dtype, device=self.device)+ update_block_global_scale_if_needed(block, self.data_type, self.group_size)if is_auto_device_mapping(self.device_map) and len(self.device_list) > 1:set_auto_device_map_for_block_with_tuning(@@ -2884,7 +2865,8 @@ class BaseCompressor(object):enable_torch_compile=self.enable_torch_compile,device=device,)- if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse+ # Call this before quantization and after applying the block wrapper.+ if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse.from auto_round.data_type.utils import update_fused_layer_global_scalesmodules = block.modules()diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex ec8f8c0c6..60cee640a 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -53,9 +53,10 @@ FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min if hasattr(torch, "float8def calculate_gparam(tensor, group_size=16, device="cpu"):- assert group_size == 16+ """Calculate global scaling factor for NVFP quantization."""+ assert group_size == 16, f"Only group_size=16 is supported, got {group_size}"if isinstance(tensor, (float, int)):- tensor_amax = torch.ones((1), device=device) * tensor+ tensor_amax = torch.tensor(tensor, device=device, dtype=torch.float32).abs()elif isinstance(tensor, torch.Tensor):tensor_amax = tensor.abs().max().to(torch.float32)global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 820ec4bde..465ea0b87 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -19,8 +19,9 @@ from typing import Listimport torchfrom torch.nn import Linear, Module+from auto_round.compressors.utils import is_nv_fpfrom auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE-from auto_round.utils import logger+from auto_round.utils import check_to_quantized, loggerdef reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: float = 0.0):@@ -318,3 +319,20 @@ def update_fused_layer_global_scales(# ---------------- MLP ----------------if _is_mlp_module(submodule):_update_global_scales([submodule.gate_proj, submodule.up_proj])+++def update_block_global_scale_if_needed(block, data_type, group_size):+ if not is_nv_fp(data_type):+ return++ from auto_round.data_type.nvfp import calculate_gparam++ # Calculate block wise weight global scale+ for _, m in block.named_modules():+ if check_to_quantized(m) and not hasattr(m, "weight_global_scale"):+ weight_global_scale = calculate_gparam(m.weight, group_size)+ setattr(m, "weight_global_scale", weight_global_scale)++ # Update fused layer global scales+ for module in block.modules():+ update_fused_layer_global_scales(module)diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 3c265f9ed..2536c0a93 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -618,6 +618,7 @@ def is_moe_layer(module: torch.nn.Module) -> bool:"DeepseekV3MoE".lower(),"Qwen2MoeSparseMoeBlock".lower(),"Qwen3MoeSparseMoeBlock".lower(),+ "Qwen3VLMoeTextSparseMoeBlock".lower(),])@@ -716,6 +717,7 @@ def get_expert_linear_names(module: torch.nn.Module) -> list[str]:"DeepseekMoE","DeepseekV2MoE","DeepseekV3MoE",+ "Qwen3VLMoeTextSparseMoeBlock",],):return ["gate_proj", "down_proj", "up_proj"]@@ -746,7 +748,15 @@ def get_expert_input_proj_names(module: torch.nn.Module) -> list[str]:return any(name.lower() in type(module).__name__.lower() for name in name_list)if module_match_name_list(- module, ["Qwen2MoeSparseMoeBlock", "Qwen3MoeSparseMoeBlock", "DeepseekMoE", "DeepseekV2MoE", "DeepseekV3MoE"]+ module,+ [+ "Qwen2MoeSparseMoeBlock",+ "Qwen3MoeSparseMoeBlock",+ "Qwen3VLMoeTextSparseMoeBlock",+ "DeepseekMoE",+ "DeepseekV2MoE",+ "DeepseekV3MoE",+ ],):# gate_proj and up_proj are input projections, down_proj is outputreturn ["gate_proj", "up_proj"]