PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
refine nvfp code, typofix (#777)
Weiweifced2ba7a8a6
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 51a64d03f..f2b44eaef 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -397,9 +397,7 @@ class AutoRound(object):self.infer_bs_coeff = 1self.enable_torch_compile = enable_torch_compileself._adjust_torch_compile(enable_torch_compile)-self._check_configs()-torch.set_printoptions(precision=3, sci_mode=True)if is_optimum_habana_available():@@ -2032,7 +2030,6 @@ class AutoRound(object):if isinstance(data_new, torch.Tensor):self.model(data_new)elif isinstance(data_new, tuple) or isinstance(data_new, list):-self.model(*data_new)else:self.model(**data_new)@@ -2499,7 +2496,7 @@ class AutoRound(object):module.act_max = act_maxelse:act_max = act_max.to(module.act_max.device)- if is_nv_fp(self.data_type): ## for nvfp per-tensor input_global_scale calculation usage+ if is_nv_fp(self.act_data_type): ## for nvfp per-tensor input_global_scale calculation usagemodule.act_max = torch.max(torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device))@@ -2736,10 +2733,13 @@ class AutoRound(object):with torch.no_grad():unwrapper_block(block, best_params)+ if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):+ from auto_round.utils import set_amax_for_all_moe_layers++ # enable moe experts act_max automatic generation for WrapperWALayer+ set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")+if self.enable_quanted_input:- if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):- # Enable moe experts act_max automatic generation for WrapperWALayer- set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")if self.low_cpu_mem_usage:block = block.to(device)clear_memory()diff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex 058c5345c..298d39e3d 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -88,7 +88,7 @@ def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, **kwargs):if global_scale is None:tensor_max = tensor.abs().max().to(torch.float32)global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)- global_scale = global_scale.to(tensor.device)+ global_scale = global_scale.to(device=tensor.device, dtype=torch.float32)qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)return qdq_res.to(orig_dtype), scale, None@@ -102,11 +102,14 @@ def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, *tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)if tensor_max is None:tensor_max = tensor.abs().max().to(torch.float32)- elif tensor_max is not None:+ else:if not isinstance(tensor_max, torch.Tensor):tensor_max = torch.tensor(tensor_max, device=tensor.device, dtype=torch.float32)+ else:+ tensor_max = tensor_max.to(device=tensor.device, dtype=torch.float32)if tensor_max.numel() != 1:- tensor_max = tensor.abs().max().to(torch.float32)+ tensor_max = tensor_max.abs().max()+global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)global_scale = global_scale.to(tensor.device)qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex f56d2fbf1..e5f195609 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -267,7 +267,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exif is_nv_fp(data_type) or is_mx_fp(data_type): ## detect nvfp & mxfp firstfrom auto_round.export.export_to_autoround.export_to_nvfp_mxfp import save_quantized_as_fp- return save_quantized_as_fp(output_dir, inplace=inplace, backend="auto_round", **kwargs)+ return save_quantized_as_fp(output_dir, inplace=inplace, backend="auto_round:llm_compressor", **kwargs)if kwargs.get("data_type", "int") == "fp" and kwargs.get("bits", 16) == 8 and kwargs.get("act_bits", 16) >= 16:from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autorounddiff --git a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pyindex a845d2396..c0d73f60a 100644--- a/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py+++ b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py@@ -47,6 +47,7 @@ __all__ = []def pack_layer(name, model, backend):if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this featurereturn@@ -157,8 +158,6 @@ def save_quantized_as_fp(output_dir, inplace=True, **kwargs):quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)quantization_config["quant_method"] = "auto-round"quantization_config["packing_format"] = backend- quantization_config["scale_format"] = ("e8m0",)- quantization_config["scale_calculation_mode"] = ("even",)tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.pyindex 678ebc487..f72fee72f 100644--- a/auto_round/export/export_to_autoround/qlinear_fp.py+++ b/auto_round/export/export_to_autoround/qlinear_fp.py@@ -149,15 +149,17 @@ class QuantLinear(nn.Module):elif torch.xpu.is_available():device = "xpu:0"- W = linear.weight.data.to(device).clone() # TODO check is nesscessory+ W = linear.weight.data.to(device).clone()if isinstance(linear, nn.Conv2d):W = W.flatten(1)if isinstance(linear, transformers.pytorch_utils.Conv1D):W = W.t()- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(linear.weight, self.group_size)+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(W, self.group_size)+ scales = scales.to(device)if self.is_nv:assert global_scale is not None and global_scale.numel() == 1+ global_scale = global_scale.to(device)scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale))diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex e6d48567e..a758daed1 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -112,7 +112,7 @@ def _handle_special_model(model):def _handle_moe_model(model):- if model.config.model_type in CONVERT_EXPERT_TO_LINEAR_MODELS:+ if hasattr(model.config, "model_type") and model.config.model_type in CONVERT_EXPERT_TO_LINEAR_MODELS:from tqdm import tqdmfrom auto_round.utils import clear_memorydiff --git a/auto_round/utils.py b/auto_round/utils.pyindex 9c94f9216..eda2a6637 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -435,7 +435,11 @@ def get_block_names(model, quant_vision=False):return block_namesdef _get_vlm_block_names(model, quant_vision=False):- if hasattr(model, "config") and model.config.model_type in SPECIAL_MULTIMODAL_BLOCK.keys():+ if (+ hasattr(model, "config")+ and hasattr(model.config, "model_type")+ and model.config.model_type in SPECIAL_MULTIMODAL_BLOCK.keys()+ ):return SPECIAL_MULTIMODAL_BLOCK.get(model.config.model_type)(model, quant_vision=quant_vision)block_names = []target_modules = []diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 9afc36fe9..4fa1ed1de 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -246,7 +246,7 @@ class WrapperLinear(torch.nn.Module):data_type=self.act_data_type,max_scale=act_max_scale,tensor_max=act_max,- global_scale=getattr(self, "weight_global_scale", None),+ global_scale=getattr(self, "input_global_scale", None),)return x, scale, zp