Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

refine global scale calculation to blockwise (#1421)

Weiweia554a107518d
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 65b4b9fc1..e855fd0ce 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -52,7 +52,7 @@ from auto_round.compressors.utils import (
set_layer_config,
)
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size
+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, update_block_global_scale_if_needed
from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG
from auto_round.formats import OutputFormat, get_formats
from auto_round.logger import logger
@@ -1274,27 +1274,6 @@ class BaseCompressor(object):
all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]
self.all_to_quantized_module_names = all_to_quantized_module_names
- if is_nv_fp(self.data_type):
- # FIXME: (yiliu30) change it to block-wise after we refactor the quantization code and
- # https://github.com/intel/auto-round/issues/1331
- materialize_model_(self.model)
- self.model.to("cpu")
- from auto_round.data_type.nvfp import calculate_gparam
- from auto_round.data_type.utils import update_fused_layer_global_scales
-
- pbar = tqdm(all_to_quantized_module_names)
- for name in pbar:
- pbar.set_description(f"Calculate weight global scale: {name}")
- m = get_module(self.model, name)
- m = convert_module_to_hp_if_necessary(m, self.amp_dtype, self.device)
- set_module(self.model, name, m)
- weight_global_scale = calculate_gparam(m.weight, self.group_size)
- setattr(m, "weight_global_scale", weight_global_scale)
-
- logger.info("Start to update fused layer global scales, it may take some time.")
- for name, module in self.model.named_modules():
- update_fused_layer_global_scales(module)
- logger.info("Finished updating fused layer global scales.")
if not (any(fmt.is_gguf() for fmt in getattr(self, "formats", [])) or self.super_bits is not None):
self._quantize_embedding_layer() # leave to gguf itself to handle
@@ -1485,7 +1464,9 @@ class BaseCompressor(object):
block = get_module(self.model, block_name)
materialize_model_(block)
block.to("cpu")
- convert_module_to_hp_if_necessary(block, dtype=self.amp_dtype, device=self.device)
+
+ block = convert_module_to_hp_if_necessary(block, dtype=self.amp_dtype, device=self.device)
+ update_block_global_scale_if_needed(block, self.data_type, self.group_size)
if is_auto_device_mapping(self.device_map) and len(self.device_list) > 1:
set_auto_device_map_for_block_with_tuning(
@@ -2884,7 +2865,8 @@ class BaseCompressor(object):
enable_torch_compile=self.enable_torch_compile,
device=device,
)
- if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse
+ # Call this before quantization and after applying the block wrapper.
+ if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse.
from auto_round.data_type.utils import update_fused_layer_global_scales
modules = block.modules()
diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.py
index ec8f8c0c6..60cee640a 100644
--- a/auto_round/data_type/nvfp.py
+++ b/auto_round/data_type/nvfp.py
@@ -53,9 +53,10 @@ FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min if hasattr(torch, "float8
def calculate_gparam(tensor, group_size=16, device="cpu"):
- assert group_size == 16
+ """Calculate global scaling factor for NVFP quantization."""
+ assert group_size == 16, f"Only group_size=16 is supported, got {group_size}"
if isinstance(tensor, (float, int)):
- tensor_amax = torch.ones((1), device=device) * tensor
+ tensor_amax = torch.tensor(tensor, device=device, dtype=torch.float32).abs()
elif isinstance(tensor, torch.Tensor):
tensor_amax = tensor.abs().max().to(torch.float32)
global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 820ec4bde..465ea0b87 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -19,8 +19,9 @@ from typing import List
import torch
from torch.nn import Linear, Module
+from auto_round.compressors.utils import is_nv_fp
from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPE
-from auto_round.utils import logger
+from auto_round.utils import check_to_quantized, logger
def reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int, val: float = 0.0):
@@ -318,3 +319,20 @@ def update_fused_layer_global_scales(
# ---------------- MLP ----------------
if _is_mlp_module(submodule):
_update_global_scales([submodule.gate_proj, submodule.up_proj])
+
+
+def update_block_global_scale_if_needed(block, data_type, group_size):
+ if not is_nv_fp(data_type):
+ return
+
+ from auto_round.data_type.nvfp import calculate_gparam
+
+ # Calculate block wise weight global scale
+ for _, m in block.named_modules():
+ if check_to_quantized(m) and not hasattr(m, "weight_global_scale"):
+ weight_global_scale = calculate_gparam(m.weight, group_size)
+ setattr(m, "weight_global_scale", weight_global_scale)
+
+ # Update fused layer global scales
+ for module in block.modules():
+ update_fused_layer_global_scales(module)
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 3c265f9ed..2536c0a93 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -618,6 +618,7 @@ def is_moe_layer(module: torch.nn.Module) -> bool:
"DeepseekV3MoE".lower(),
"Qwen2MoeSparseMoeBlock".lower(),
"Qwen3MoeSparseMoeBlock".lower(),
+ "Qwen3VLMoeTextSparseMoeBlock".lower(),
]
)
@@ -716,6 +717,7 @@ def get_expert_linear_names(module: torch.nn.Module) -> list[str]:
"DeepseekMoE",
"DeepseekV2MoE",
"DeepseekV3MoE",
+ "Qwen3VLMoeTextSparseMoeBlock",
],
):
return ["gate_proj", "down_proj", "up_proj"]
@@ -746,7 +748,15 @@ def get_expert_input_proj_names(module: torch.nn.Module) -> list[str]:
return any(name.lower() in type(module).__name__.lower() for name in name_list)
if module_match_name_list(
- module, ["Qwen2MoeSparseMoeBlock", "Qwen3MoeSparseMoeBlock", "DeepseekMoE", "DeepseekV2MoE", "DeepseekV3MoE"]
+ module,
+ [
+ "Qwen2MoeSparseMoeBlock",
+ "Qwen3MoeSparseMoeBlock",
+ "Qwen3VLMoeTextSparseMoeBlock",
+ "DeepseekMoE",
+ "DeepseekV2MoE",
+ "DeepseekV3MoE",
+ ],
):
# gate_proj and up_proj are input projections, down_proj is output
return ["gate_proj", "up_proj"]