PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Enable mxfp exporting (#649)
Weiwei64b8afee0922
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 3c0b07247..854f3f460 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -64,7 +64,10 @@ from auto_round.utils import (infer_bits_by_data_type,init_cache,is_debug_mode,+ is_mx_fp,+ is_nv_fp,is_optimum_habana_available,+ is_standard_fp,llm_load_model,logger,mv_module_from_gpu,@@ -480,16 +483,10 @@ class AutoRound(object):"And please save the quantized model to fake format as real deployment is not supported currently")- if "mx_fp" in self.data_type or "nv_fp" in self.data_type:- logger.warning(- "please save the quantized model to fake format "- "as real deployment is not supported for mx_fp/nv_fp datatype currently"- )-- if "mx_fp" in self.data_type and self.group_size != 32:+ if is_mx_fp(self.data_type) and self.group_size != 32:logger.warning("mx_fp should only support group_size of 32 in real deployment")- if "nv_fp" in self.data_type and (self.group_size != 16):+ if is_nv_fp(self.data_type) and (self.group_size != 16):logger.warning("nv_fp should only support group_size of 16 in real deployment")if self.nsamples < self.gradient_accumulate_steps * self.batch_size:@@ -620,9 +617,20 @@ class AutoRound(object):)if enable_awq:formats[index] = format.replace("auto_round", "auto_round:auto_awq")- if "fp8" in self.data_type:+ if is_nv_fp(self.data_type) or is_mx_fp(self.data_type) or is_standard_fp(self.data_type):format = format.replace("auto_round", f"auto_round:{self.data_type}")formats[index] = format+ elif format == "llmcompressor":+ from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported++ if check_compressed_tensors_supported() and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):+ format = format.replace("llmcompressor", f"llmcompressor:{self.data_type}")+ formats[index] = format+ elif not is_standard_fp(self.data_type):+ logger.error(+ "Currently, the llmcompressor format only supports MXFP/NVFP/StandardFP8. "+ "Please change format to fake or auto_round etc."+ )# Remove duplicates from formats listdef remove_duplicates(lst):@@ -651,10 +659,12 @@ class AutoRound(object):if format == "fake":return formatformat = format.replace("q*_", f"q{self.bits}_")- # only support to export afp8+ # only support to export afp8/nv_fpif self.act_bits <= 8:- if "fp8" not in self.act_data_type or self.act_dynamic:+ if not is_standard_fp(self.act_data_type) or self.act_dynamic:if format == "llmcompressor":+ if is_nv_fp(self.act_data_type):+ return formatbits, group_size, sym, act_bits = 8, -1, True, 8assert (self.bits == bits@@ -667,10 +677,10 @@ class AutoRound(object):f" W{self.bits}A{self.act_bits} model, but got bits={self.bits},"f" group_size={self.group_size}, sym={self.sym}, act_bits={self.act_bits}")- elif format != "fake":+ elif format != "fake" and not is_nv_fp(format):logger.warning("Currently only support to export auto_round format quantized model"- " with fp8 dtype activation for activation quantization."+ " with fp8 or nv_fp4 dtype activation for activation quantization."" Change format to fake and save.")format = "fake"@@ -1130,6 +1140,7 @@ class AutoRound(object):m.to("cpu")except RuntimeError as e:cuda_error_msg = traceback.format_exc()+ m = m.orig_layer if hasattr(m, "orig_layer") else mtry:logger.warning("Out of VRAM, falling back to CPU.")m.to("cpu")@@ -1191,6 +1202,21 @@ class AutoRound(object):all_to_quantized_module_names: list[str] = [n for n, m in self.model.named_modules() if check_to_quantized(m)]+ if is_nv_fp(self.data_type):+ from auto_round.data_type.nvfp import calculate_gparam+ from auto_round.data_type.utils import update_fused_layer_global_scales++ pbar = tqdm(all_to_quantized_module_names)+ for name in pbar:+ pbar.set_description(f"Calculate weight global scale: {name}")+ m = get_module(self.model, name)+ weight_global_scale = calculate_gparam(m.weight, self.group_size)+ setattr(m, "weight_global_scale", weight_global_scale)++ modules = list(self.model.modules())+ for module in tqdm(modules, desc="Update weight global scale for fuse module"):+ update_fused_layer_global_scales(module)+has_gguf_k = any("gguf" in fmt and "k" in fmt for fmt in getattr(self, "formats", []))self.quantize_embedding_layer()@@ -1318,6 +1344,11 @@ class AutoRound(object):if self.device_map is not None:accelerate.hooks.remove_hook_from_submodules(block)+ if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):+ from auto_round.utils import set_amax_for_all_moe_layers++ # enable moe experts act_max automatic generation for linears+ set_amax_for_all_moe_layers(block, attr_name="act_max")# Normalize imatrix and quantize layersfor _, m in block.named_modules():if hasattr(m, "imatrix"):@@ -1374,7 +1405,13 @@ class AutoRound(object):formats = self.formatsif (len(formats) == 1- and ("awq" in formats[0] or "gptq" in formats[0] or "auto_round" in formats[0] or "gguf" in formats[0])+ and (+ "awq" in formats[0]+ or "gptq" in formats[0]+ or "auto_round" in formats[0]+ or "gguf" in formats[0]+ or "llmcompressor" in formats[0]+ )and self.inplace):self.is_packing_immediate = True@@ -2217,16 +2254,23 @@ class AutoRound(object):logger.info(dump_info)def register_act_max_hook(self, model):-def get_act_max_hook(module, input, output):if isinstance(input, (tuple, list)):input = input[0]+ if input.numel() == 0:+ return # as no needs for act_max updateinput, _, _ = reshape_pad_tensor_by_group_size(input, self.act_group_size)act_max = torch.max(torch.abs(input), dim=-1).values- if not hasattr(module, "act_max"):+ if not hasattr(module, "act_max") or module.act_max.numel() == 0:module.act_max = act_maxelse:- module.act_max = torch.max(act_max.to(module.act_max.device), module.act_max)+ act_max = act_max.to(module.act_max.device)+ if is_nv_fp(self.data_type): ## for nvfp per-tensor input_global_scale calculation usage+ module.act_max = torch.max(+ torch.tensor([act_max.max(), module.act_max.max()], device=act_max.device)+ )+ else:+ module.act_max = torch.max(act_max, module.act_max)hook_handles = []@@ -2323,7 +2367,12 @@ class AutoRound(object):quantized_layer_names, unquantized_layer_names = wrapper_block(block, self.enable_minmax_tuning, self.enable_norm_bias_tuning, device=self.device)+ if is_nv_fp(self.data_type): # enable qkv and moe structure global_scale fuse+ from auto_round.data_type.utils import update_fused_layer_global_scales+ modules = block.modules()+ for module in modules:+ update_fused_layer_global_scales(module)round_params = []minmax_params = []for n, m in block.named_modules():@@ -2446,6 +2495,11 @@ class AutoRound(object):with torch.no_grad():unwrapper_block(block, best_params)if self.enable_quanted_input:+ if is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats):+ from auto_round.utils import set_amax_for_all_moe_layers++ # enable moe experts act_max automatic generation for WrapperWALayer+ set_amax_for_all_moe_layers(block, attr_name="orig_layer.act_max")if self.low_cpu_mem_usage:block = block.to(device)clear_memory()@@ -2613,6 +2667,8 @@ class AutoRound(object):"Support for exporting activation quantization is limited. ""Please ensure that your configuration is supported.")+ if format == "llmcompressor" and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):+ format = format.replace("llmcompressor", f"llmcompressor:{self.data_type}")from auto_round.export import EXPORT_FORMAT@@ -2649,6 +2705,7 @@ class AutoRound(object):layer_config=self.layer_config,inplace=inplace,bits=self.bits,+ act_bits=self.act_bits,group_size=self.group_size,sym=self.sym,iters=self.iters,@@ -2660,6 +2717,7 @@ class AutoRound(object):tokenizer=self.tokenizer,supported_types=self.supported_types,data_type=self.data_type,+ act_data_type=self.act_data_type,serialization_dict=serialization_dict,backend=backend,to_quant_block_names=self.to_quant_block_names,diff --git a/auto_round/data_type/__init__.py b/auto_round/data_type/__init__.pyindex 03e9160e1..ed446f314 100644--- a/auto_round/data_type/__init__.py+++ b/auto_round/data_type/__init__.py@@ -17,6 +17,6 @@ import auto_round.data_type.mxfpimport auto_round.data_type.fp8from auto_round.data_type.register import QUANT_FUNC_WITH_DTYPEimport auto_round.data_type.w4fp8-from auto_round.data_type.utils import get_quant_func+from auto_round.data_type.utils import get_quant_func, update_fused_layer_global_scalesimport auto_round.data_type.nvfpimport auto_round.data_type.ggufdiff --git a/auto_round/data_type/nvfp.py b/auto_round/data_type/nvfp.pyindex a5b7bfebc..058c5345c 100644--- a/auto_round/data_type/nvfp.py+++ b/auto_round/data_type/nvfp.py@@ -57,6 +57,16 @@ FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max if hasattr(torch, "float8FLOAT8_E4M3_MIN = torch.finfo(torch.float8_e4m3fn).min if hasattr(torch, "float8_e4m3fn") else -448+def calculate_gparam(tensor, group_size=16, device="cpu"):+ assert group_size == 16+ if isinstance(tensor, (float, int)):+ tensor_amax = torch.ones((1), device=device) * tensor+ elif isinstance(tensor, torch.Tensor):+ tensor_amax = tensor.abs().max().to(torch.float32)+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_amax)+ return global_scale++def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):assert global_scale.dtype == torch.float32assert x.ndim == 2@@ -68,22 +78,26 @@ def ref_nvfp4_quant(x, global_scale, block_size=16, v=0):output_scale = get_reciprocal(scale * get_reciprocal(global_scale))scaled_x = x.to(torch.float32) * output_scale + vclipped_x = torch.clamp(scaled_x, -6.0, 6.0)- return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), output_scale+ return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale@register_dtype("nv_fp4")-def nv_fp4(tensor, bits=4, group_size=16, v=0, **kwargs):+def nv_fp4(tensor, bits=4, group_size=16, v=0, global_scale=None, **kwargs):orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- tensor_max = tensor.abs().max().to(torch.float32)- global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)- qdq_res, output_scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)+ if global_scale is None:+ tensor_max = tensor.abs().max().to(torch.float32)+ global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)+ global_scale = global_scale.to(tensor.device)+ qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)- return qdq_res.to(orig_dtype), output_scale, None+ return qdq_res.to(orig_dtype), scale, None@register_dtype("nv_fp4_with_static_gs")def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, **kwargs):+ if tensor is None or tensor.numel() == 0:+ return tensor, None, Noneorig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)if tensor_max is None:@@ -94,9 +108,10 @@ def nv_fp4_with_static_gs(tensor, bits=4, group_size=16, v=0, tensor_max=None, *if tensor_max.numel() != 1:tensor_max = tensor.abs().max().to(torch.float32)global_scale = FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)- qdq_res, output_scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)+ global_scale = global_scale.to(tensor.device)+ qdq_res, scale = ref_nvfp4_quant(tensor, global_scale, group_size, v)qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)- return qdq_res.to(orig_dtype), output_scale, None+ return qdq_res.to(orig_dtype), scale, NoneFLOAT8_UE5M3_MAX = 114688@@ -109,7 +124,7 @@ def float_to_e5m3_frexp(x: torch.Tensor) -> torch.Tensor:mask = x > 0x_masked = x[mask]- # 正常数:x >= 2^-14+ # normal number: x >= 2^-14normal_mask = x_masked >= 2**-14x_normal = x_masked[normal_mask]mantissa, exponent = torch.frexp(x_normal)@@ -178,6 +193,7 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):assert (not isinstance(global_scale, torch.Tensor)) or global_scale.dtype == torch.float32assert x.ndim == 2m, n = x.shape+ device = x.deviceif isinstance(max_scale, torch.Tensor):max_scale = max_scale.unsqueeze(dim=-1).to(x.device)vec_max = torch.max(torch.abs(x), dim=-1, keepdim=True)[0].to(torch.float32) * max_scale@@ -185,19 +201,16 @@ def ref_fp4_quant(x, global_scale, block_size=16, v=0, max_scale=1.0):scale = torch.clip(scale, 0, FLOAT8_UE5M3_MAX)scale = cast_to_ue5m3_ste(scale).to(torch.float32)output_scale = get_reciprocal(scale * get_reciprocal(global_scale))-scaled_x = x.to(torch.float32) * output_scale + vclipped_x = torch.clamp(scaled_x, -6.0, 6.0)- return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), output_scale+ return (cast_to_fp4_ste(clipped_x) * get_reciprocal(output_scale)).reshape(m, n), scale@register_dtype("fp4_v2_with_global_scale")def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=None, max_scale=1.0, **kwargs):assert group_size == 32 or group_size == 16orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)-if tensor_max is None:tensor_max = tensor.abs().max().to(torch.float32)elif tensor_max is not None:@@ -206,21 +219,20 @@ def fp4_v2_with_global_scale(tensor, bits=4, group_size=16, v=0, tensor_max=Noneif tensor_max.numel() != 1:tensor_max = tensor.abs().max().to(torch.float32)global_scale = FLOAT8_UE5M3_MAX * FLOAT4_E2M1_MAX * get_reciprocal(tensor_max)- qdq_res, output_scale = ref_fp4_quant(tensor, global_scale, group_size, v)+ qdq_res, scale = ref_fp4_quant(tensor, global_scale, group_size, v)qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)- return qdq_res.to(orig_dtype), output_scale, None+ return qdq_res.to(orig_dtype), scale, None@register_dtype("fp4_v2")def fp4_v2(tensor, bits=4, group_size=32, v=0, max_scale=1.0, **kwargs):assert group_size == 32 or group_size == 16orig_dtype = tensor.dtypetensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)global_scale = 1.0- qdq_res, output_scale = ref_fp4_quant(tensor, global_scale, group_size, v, max_scale)+ qdq_res, scale = ref_fp4_quant(tensor, global_scale, group_size, v, max_scale)qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)- return qdq_res.to(orig_dtype), output_scale, None+ return qdq_res.to(orig_dtype), scale, Noneif __name__ == "__main__":diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 09178ed94..371104b9c 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -12,9 +12,12 @@# See the License for the specific language governing permissions and# limitations under the License.+import mathfrom functools import lru_cache+from typing import Listimport torch+from torch.nn import Linear, Modulefrom auto_round.data_type.register import QUANT_FUNC_WITH_DTYPEfrom auto_round.utils import logger@@ -23,8 +26,8 @@ from auto_round.utils import loggerdef reshape_pad_tensor_by_group_size(data: torch.Tensor, group_size: int):"""Reshapes and pads the tensor to ensure that it can be quantized in groups of `group_size`.- This function adjusts t- he input tensor's shape so that its last dimension is a multiple+ This function adjusts the+ input tensor's shape so that its last dimension is a multipleof the specified `group_size`. If padding is required, it adds padding to the tensorto achieve this. If the tensor's last dimension is already divisible by `group_size`,no padding is applied.@@ -111,7 +114,7 @@ def get_quant_func(dtype, bits, sym):if key in QUANT_FUNC_WITH_DTYPE.keys():return QUANT_FUNC_WITH_DTYPE[key], key- ##need to add bits+ ##need to add bits and sym infosif sym:key = dtype + str(bits) + "_sym"else:@@ -120,14 +123,6 @@ def get_quant_func(dtype, bits, sym):if key in QUANT_FUNC_WITH_DTYPE.keys():return QUANT_FUNC_WITH_DTYPE[key], key- if sym:- key = dtype + "_sym"- else:- key = dtype + "_asym"-- if key in QUANT_FUNC_WITH_DTYPE.keys():- return QUANT_FUNC_WITH_DTYPE[key], key-if sym:key = dtype + str(bits)else:@@ -237,3 +232,74 @@ def get_gaudi_fp8_ste_func():fn = float8_e4m3fn_stelogger.warning_once("Using CUDA/CPU STE for FP8")return fn+++# please refer from https://github.com/vllm-project/llm-compressor/blob/+# 29f4d5644b48e9c8ebb7e36d5be9f7c92747ceb7/src/llmcompressor/modifiers/utils/helpers.py#L11+def update_fused_layer_global_scales(submodule: torch.nn.Module, base_name="weight"):+ """+ When running NVFP4 quantization, update the global scale+ such that q,k,v layers are treated as one tensor with the same+ global_scale and gate_proj/up_proj layers are treated as one tensor+ with the same global scale. This is requirement currently being set+ by vLLM and may be removed in the future OR potentially make it+ an optional step.++ :param model: model to quantize+ base_name: op name for fuse usage, option: weight, input+ """+ global_scale_name = f"{base_name}_global_scale"+ max_value_tensor = torch.ones(1, device="cpu", dtype=torch.float32) * torch.finfo(torch.float32).max++ def _is_attention_module(module: Module):+ return "attention" in module.__class__.__name__.lower() and (+ hasattr(module, "k_proj") or hasattr(module, "v_proj") or hasattr(module, "qkv_proj")+ )++ def _is_mlp_module(module: Module):+ return "mlp" in module.__class__.__name__.lower() and (+ hasattr(module, "gate_proj") or hasattr(module, "up_proj")+ )++ if _is_attention_module(submodule):+ # already fused/treated as one layer+ if hasattr(submodule, "qkv_proj"):+ return+ global_scale = torch.min(+ torch.cat(+ (+ getattr(submodule.q_proj, global_scale_name, max_value_tensor).reshape(1),+ getattr(submodule.k_proj, global_scale_name, max_value_tensor).reshape(1),+ getattr(submodule.v_proj, global_scale_name, max_value_tensor).reshape(1),+ )+ )+ ).reshape([1])++ if math.isclose(global_scale.data, max_value_tensor.data, rel_tol=1e-9):+ return+ if hasattr(submodule.q_proj, global_scale_name):+ setattr(submodule.q_proj, global_scale_name, global_scale.clone())+ if hasattr(submodule.k_proj, global_scale_name):+ setattr(submodule.k_proj, global_scale_name, global_scale.clone())+ if hasattr(submodule.v_proj, global_scale_name):+ setattr(submodule.v_proj, global_scale_name, global_scale.clone())++ del global_scale++ if _is_mlp_module(submodule):+ global_scale = torch.min(+ torch.cat(+ (+ getattr(submodule.gate_proj, global_scale_name, max_value_tensor).reshape(1),+ getattr(submodule.up_proj, global_scale_name, max_value_tensor).reshape(1),+ )+ )+ ).reshape([1])+ if math.isclose(global_scale.data, max_value_tensor.data, rel_tol=1e-9):+ return+ if hasattr(submodule.gate_proj, global_scale_name):+ setattr(submodule.gate_proj, global_scale_name, global_scale.clone())+ if hasattr(submodule.up_proj, global_scale_name):+ setattr(submodule.up_proj, global_scale_name, global_scale.clone())++ del global_scalediff --git a/auto_round/export/__init__.py b/auto_round/export/__init__.pyindex 99c19822a..5956238d6 100644--- a/auto_round/export/__init__.py+++ b/auto_round/export/__init__.py@@ -83,3 +83,10 @@ def _save_quantized_as_llmcompressor(*args, **kwargs):from auto_round.export.export_to_llmcompressor.export import save_quantized_as_llmcompressorreturn save_quantized_as_llmcompressor(*args, **kwargs)+++@register_layer_packing("llmcompressor")+def _packing_layer_with_llmcompressor(*args, **kwargs):+ from auto_round.export.export_to_llmcompressor.export import pack_layer++ return pack_layer(*args, **kwargs)diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 3e72c9590..22ffe967d 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -33,27 +33,14 @@ from auto_round.utils import (filter_quantization_config,get_autogptq_packing_qlinear,get_module,+ is_mx_fp,+ is_nv_fp,+ is_standard_fp,logger,set_module,)--def check_neq_config(config, data_type, bits, group_size, sym):- """- Checks if the provided configuration parameters are not equal to the values in the config dictionary.-- Args:- config (dict): A dictionary containing the configuration parameters.- data_type (str): The expected data type.- bits (int): The expected number of bits.- group_size (int): The expected group size.- sym (bool): The expected symmetry flag.-- Returns:- list: A list of strings indicating which configuration parameters do not match.- """- expected_config = {"data_type": data_type, "bits": bits, "group_size": group_size, "sym": sym}- return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]+from .utils import check_neq_configdef dynamic_import_quant_linear_for_packing(backend, bits, group_size, sym, act_bits=16):@@ -160,6 +147,16 @@ def pack_layer(layer_name, model, backend):Returns:None: The function modifies the model in place."""+ if is_nv_fp(backend) or is_mx_fp(backend):+ from auto_round.export.export_to_autoround.export_to_fp import pack_layer++ return pack_layer(layer_name, model, backend)++ if is_standard_fp(backend):+ from auto_round.export.export_to_autoround.export_to_fp8_woq import pack_layer++ return pack_layer(layer_name, model, backend)+layer = get_module(model, layer_name)if hasattr(layer, "orig_layer"):layer = layer.orig_layer@@ -167,11 +164,6 @@ def pack_layer(layer_name, model, backend):if not isinstance(layer, SUPPORTED_LAYER_TYPES): ##already packedreturn- if "fp8" in backend:- from auto_round.export.export_to_autoround.export_to_fp8_woq import pack_layer-- return pack_layer(layer_name, model, backend)-if int(layer.act_bits) <= 8:return pack_qact_layer(layer_name, model)@@ -267,7 +259,13 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exRaises:ValueError: If the backend is not supported."""- if "fp8" in kwargs.get("data_type", None) and kwargs.get("act_bits", 16) >= 16:+ data_type = kwargs.get("data_type", None)+ if is_nv_fp(data_type) or is_mx_fp(data_type): ## detect nvfp & mxfp first+ from auto_round.export.export_to_autoround.export_to_fp import save_quantized_as_fp++ return save_quantized_as_fp(output_dir, inplace=inplace, backend="auto_round", **kwargs)++ if is_standard_fp(data_type) and kwargs.get("act_bits", 16) >= 16:from auto_round.export.export_to_autoround.export_to_fp8_woq import save_quantized_as_autoroundreturn save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)@@ -314,6 +312,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exlayer_config[layer_name],data_type=quantization_config["data_type"],bits=quantization_config["bits"],+ act_bits=quantization_config["act_bits"],group_size=quantization_config["group_size"],sym=quantization_config["sym"],)diff --git a/auto_round/export/export_to_autoround/export_to_fp.py b/auto_round/export/export_to_autoround/export_to_fp.pynew file mode 100644index 000000000..a3c724ad9--- /dev/null+++ b/auto_round/export/export_to_autoround/export_to_fp.py@@ -0,0 +1,294 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import copy+import inspect+import json+import os+from concurrent.futures import ThreadPoolExecutor++import threadpoolctl as tctl+import torch+import torch.nn as nn+import transformers+from tqdm import tqdm++from auto_round.utils import (+ SUPPORTED_LAYER_TYPES,+ check_start_with_block_name,+ check_to_quantized,+ filter_quantization_config,+ get_block_names,+ get_module,+ is_mx_fp,+ is_nv_fp,+ logger,+ set_amax_for_all_moe_layers,+ set_module,+)+from auto_round.wrapper import WrapperWALayer++from .qlinear_fp import QuantLinear+from .utils import check_neq_config++__all__ = [+ "pack_layer",+ "save_quantized_as_fp",+]+++def pack_layer(name, model, backend):+ if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this feature+ return+ layer = get_module(model, name)++ if not isinstance(layer, SUPPORTED_LAYER_TYPES) and not isinstance(layer, WrapperWALayer): ##already packed+ return++ if isinstance(layer, WrapperWALayer): # revert WrapperWALayer for offline usage+ wp_layer = layer+ layer = wp_layer.orig_layer+ set_module(model, name, layer)++ data_type = layer.data_type+ act_bits = layer.act_bits+ act_data_type = layer.act_data_type+ bits = layer.bits+ if bits > 8:+ return+ group_size = layer.group_size+ sym = layer.sym+ device = layer.weight.device++ if is_nv_fp(act_data_type) and act_bits <= 8:+ if not getattr(layer, "input_global_scale", None):+ assert hasattr(layer, "act_max")+ from auto_round.data_type.nvfp import calculate_gparam++ input_global_scale = calculate_gparam(layer.act_max, layer.group_size, "cpu") # , model.device+ setattr(layer, "input_global_scale", input_global_scale)+ delattr(layer, "act_max")++ # QuantLinear = get_fp_qlinear(backend, bits, group_size, sym)++ if isinstance(layer, nn.Linear):+ in_features = layer.in_features+ out_features = layer.out_features+ elif isinstance(layer, nn.Conv2d):+ in_features = layer.in_channels+ out_features = layer.out_channels+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):+ in_features = layer.weight.shape[0]+ out_features = layer.weight.shape[1]++ bias = layer.bias is not None+ ##bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?+ new_layer = QuantLinear( ##pylint: disable=E1123+ bits,+ group_size,+ in_features,+ out_features,+ bias,+ weight_dtype=layer.weight.dtype,+ sym=sym,+ data_type=data_type,+ act_bits=act_bits,+ act_data_type=act_data_type,+ )++ new_layer.device = device+ set_module(model, name, new_layer)+ qlayer = new_layer+ scale = layer.scale+ global_scale = getattr(layer, "weight_global_scale", None)+ input_global_scale = getattr(layer, "input_global_scale", None)+ # zero = layer.zp+ # so far can only pack layer on CPU+ qlayer.to("cpu")+ layer, scale = layer.to("cpu"), scale.to("cpu")+ qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale)+ ## no zeros to handle, as mxfp not support asym quantization+ qlayer.to(device)+++def save_quantized_as_fp(output_dir, inplace=True, **kwargs):+ """+ Saves a quantized model of mxfp/nvfp data_type in the auto-round format.++ Args:+ output_dir (str): The directory where the quantized model will be saved.+ inplace (bool, optional): If True, modifies the model in place. Otherwise, creates a deepcopy of the model.+ Default is True.+ backend (str, optional): The backend to be used for quantization.+ Default is "autoround:exllamav2".+ **kwargs: Additional keyword arguments including:+ - model (nn.Module): The model to be quantized.+ - layer_config (dict): The layer configuration for each layer.+ - serialization_dict (dict): The serialization configuration.+ - tokenizer (Tokenizer, optional): The tokenizer to be saved.++ Returns:+ None++ Raises:+ ValueError: If the backend is not supported.+ """+ model = kwargs["model"]+ backend = kwargs.get("backend", None)+ bits = kwargs.get("bits", None)+ data_type = kwargs.get("data_type", None)+ act_bits = kwargs.get("act_bits", None)+ act_data_type = kwargs.get("act_data_type", None)+ safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]+ if not inplace:+ model = copy.deepcopy(model.to("cpu"))+ layer_config = kwargs["layer_config"]+ quantization_config = kwargs["serialization_dict"]+ quantization_config["block_name_to_quantize"] = quantization_config.pop("to_quant_block_names", None)+ quantization_config["quant_method"] = "auto-round"+ quantization_config["packing_format"] = backend+ quantization_config["scale_format"] = ("e8m0",)+ quantization_config["scale_calculation_mode"] = ("even",)++ tokenizer = kwargs.get("tokenizer", None)+ processor = kwargs.get("processor", None)+ extra_config = {}++ if act_bits <= 8:+ # revert WrapperWALayer for offline usage+ for n, m in model.named_modules():+ if isinstance(m, WrapperWALayer):+ orig_layer = m.orig_layer+ if not getattr(orig_layer, "input_global_scale", None):+ assert hasattr(orig_layer, "act_max")+ from auto_round.data_type.nvfp import calculate_gparam++ input_global_scale = calculate_gparam(orig_layer.act_max, orig_layer.group_size, model.device)+ setattr(orig_layer, "input_global_scale", input_global_scale)+ delattr(orig_layer, "act_max")+ set_module(model, n, orig_layer)++ # update input_global_scale+ from auto_round.data_type.utils import update_fused_layer_global_scales++ modules = list(model.modules())+ for module in tqdm(modules, desc="Update input global scale for fuse modules"):+ update_fused_layer_global_scales(module, base_name="input")++ block_name_to_quantize = quantization_config["block_name_to_quantize"]+ if isinstance(block_name_to_quantize, str):+ block_name_to_quantize = block_name_to_quantize.split(",")+ elif isinstance(block_name_to_quantize, list):+ for i in range(len(block_name_to_quantize)):+ block_name_to_quantize[i] = os.path.commonprefix(block_name_to_quantize[i]).rstrip(".")++ for layer_name in layer_config:+ if (+ not layer_config[layer_name]["in_blocks"] and layer_config[layer_name]["bits"] <= 8+ ): ##lm head ##TODO fix act and so on+ extra_config[layer_name] = {}+ extra_config[layer_name]["bits"] = layer_config[layer_name]["bits"]+ extra_config[layer_name]["data_type"] = layer_config[layer_name]["data_type"]+ extra_config[layer_name]["group_size"] = layer_config[layer_name]["group_size"]+ extra_config[layer_name]["sym"] = layer_config[layer_name]["sym"]+ elif layer_config[layer_name]["in_blocks"] or (+ block_name_to_quantize is not None and check_start_with_block_name(layer_name, block_name_to_quantize)+ ):+ neq_keys = check_neq_config(+ layer_config[layer_name],+ data_type=quantization_config["data_type"],+ bits=quantization_config["bits"],+ act_bits=quantization_config["act_bits"],+ group_size=quantization_config["group_size"],+ sym=quantization_config["sym"],+ )+ if len(neq_keys) > 0:+ extra_config[layer_name] = {}+ for key in neq_keys:+ if layer_config[layer_name][key] is not None:+ extra_config[layer_name][key] = layer_config[layer_name][key]+ if len(extra_config) > 0:+ quantization_config["extra_config"] = extra_config+ names = list(layer_config.keys())+ max_workers = 1+ if not torch.cuda.is_available() or not torch.xpu.is_available():+ max_workers = 2 ## 2 with cuda packing will cause hang occasionally+ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ with tqdm(total=len(names), leave=True) as pbar:++ def wrapper(name):+ pbar.set_description(f"packing {name}")+ with tctl.threadpool_limits(limits=1):+ pack_layer(name, model, backend)+ pbar.update(1)++ for _ in executor.map(wrapper, names):+ pass+ filter_quantization_config(quantization_config)++ if hasattr(model, "config"):+ model.config.quantization_config = quantization_config+ if output_dir is None:+ return model++ if output_dir is None:+ model.tokenizer = tokenizer+ return model+ if os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")+ if tokenizer is not None:+ tokenizer.save_pretrained(output_dir)++ if processor is not None:+ processor.save_pretrained(output_dir)++ dtype = None+ save(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)++ return model+++def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True, dtype=None):+ """Save model state dict and configs.++ Args:+ model (`nn.Module`):+ Model to be saved. The model can be wrapped or unwrapped.+ save_dir (`str`):+ Directory to which to save. Will be created if it doesn't exist.+ max_shard_size (`str`, defaults to `"10GB"`):+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).+ <Tip warning={true}>++ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard+ which will be bigger than `max_shard_size`.++ </Tip>+ safe_serialization (`bool`, defaults to `True`):+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).+ """+ os.makedirs(save_dir, exist_ok=True)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ config_path = os.path.join(save_dir, "config.json")+ if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):+ with open(config_path, "r") as file:+ data = json.load(file)+ data["torch_dtype"] = str(dtype).split(".")[-1]+ with open(config_path, "w") as file:+ json.dump(data, file, indent=2)+ config_file = "quantization_config.json"+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:+ json.dump(model.config.quantization_config, f, indent=2)diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8_woq.pyindex 5b6a4c400..b543c4390 100644--- a/auto_round/export/export_to_autoround/export_to_fp8_woq.py+++ b/auto_round/export/export_to_autoround/export_to_fp8_woq.py@@ -32,23 +32,7 @@ from auto_round.utils import (set_module,)--def check_neq_config(config, data_type, bits, group_size, sym):- """- Checks if the provided configuration parameters are not equal to the values in the config dictionary.-- Args:- config (dict): A dictionary containing the configuration parameters.- data_type (str): The expected data type.- bits (int): The expected number of bits.- group_size (int): The expected group size.- sym (bool): The expected symmetry flag.-- Returns:- list: A list of strings indicating which configuration parameters do not match.- """- expected_config = {"data_type": data_type, "bits": bits, "group_size": group_size, "sym": sym}- return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]+from .utils import check_neq_configclass FP8WOQLinear(torch.nn.Module):@@ -199,6 +183,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round",layer_config[layer_name],data_type=quantization_config["data_type"],bits=quantization_config["bits"],+ act_bits=quantization_config["act_bits"],group_size=quantization_config["group_size"],sym=quantization_config["sym"],)diff --git a/auto_round/export/export_to_autoround/qlinear_fp.py b/auto_round/export/export_to_autoround/qlinear_fp.pynew file mode 100644index 000000000..bd70a43ba--- /dev/null+++ b/auto_round/export/export_to_autoround/qlinear_fp.py@@ -0,0 +1,258 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++# Copyright (c) 2021 - present / Neuralmagic, Inc. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import math+from logging import getLogger++import numpy as np+import torch+import torch.nn as nn+import transformers++from auto_round.data_type.mxfp import FP32_EXPONENT_BIAS, FP32_MIN_NORMAL+from auto_round.data_type.nvfp import cast_to_fp4, get_reciprocal+from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.utils import is_mx_fp, is_nv_fp, is_standard_fp++# from auto_round.utils import get_weight_compress_dtype+logger = getLogger(__name__)+E8M0_EXPONENT_BIAS = 127+E8M0_EXPONENT_NAN_VAL = 255++__all__ = ["QuantLinear"]++FLOAT_TO_E2M1 = [+ 0.0,+ 0.5,+ 1.0,+ 1.5,+ 2.0,+ 3.0,+ 4.0,+ 6.0,+]+++class QuantLinear(nn.Module):+ """+ MXFP quantized linear layer.+ """++ QUANT_TYPE = "MXFP"++ def __init__(+ self, bits, group_size, infeatures, outfeatures, bias, trainable=False, data_type="mx_fp8e4m3", **kwargs+ ):+ super().__init__()+ if bits not in [4, 8]:+ raise NotImplementedError("Only 4,8 bits are supported.")+ if infeatures % 32 != 0 or outfeatures % 32 != 0:+ raise NotImplementedError("in_feature and out_feature must be divisible by 32.")+ self.is_mx = is_mx_fp(data_type)+ self.is_nv = is_nv_fp(data_type)+ if self.is_mx and group_size != 32:+ raise NotImplementedError("Only group_size 32 are supported for mxfp.")+ if self.is_nv and group_size not in [16, 32]:+ raise NotImplementedError("Only group_size 16 are supported for nvfp.")+ self.infeatures = infeatures+ self.outfeatures = outfeatures+ self.bits = bits+ self.data_type = data_type+ self.sym = kwargs.get("sym", True)+ self.group_size = group_size if group_size != -1 else infeatures+ self.maxq = 2**self.bits - 1+ self.act_bits = kwargs.get("act_bits", None)++ weight_name = "weight" if self.bits == 8 and self.is_mx else "weight_packed"+ ## TODO check the dtype of weight_packed and weight_scale+ self.register_buffer(+ weight_name,+ torch.zeros((infeatures // 32 * self.bits, outfeatures), dtype=torch.int32),+ )+ if not self.sym:+ ## TODO Currently only sym quant is supported for mxfp dtype. Is weight_zero_point param necessary?+ self.register_buffer(+ "weight_zero_point",+ torch.zeros(+ (+ math.ceil(infeatures / self.group_size),+ outfeatures // 32 * self.bits,+ ),+ dtype=torch.int32,+ ),+ )+ self.register_buffer(+ "weight_scale",+ torch.zeros(+ (math.ceil(infeatures / self.group_size), outfeatures),+ dtype=torch.float16, ## TODO update to correct scale dtype for different bits+ ),+ )+ if self.is_nv and self.bits == 4:+ self.register_buffer(+ "weight_global_scale",+ torch.zeros(+ (1),+ dtype=torch.float32,+ ),+ )+ if self.is_nv and self.act_bits <= 8:+ self.register_buffer(+ "input_global_scale",+ torch.zeros(+ (1),+ dtype=torch.float32,+ ),+ )+ if bias:+ self.register_buffer("bias", torch.zeros((outfeatures), dtype=torch.float16))+ else:+ self.bias = None++ self.trainable = trainable++ def post_init(self):+ pass++ def pack(self, linear, scales, zeros=None, g_idx=None, global_scale=None, input_global_scale=None):+ if linear.bias is not None:+ self.bias = linear.bias.clone().half()+ device = "cpu"+ if torch.cuda.is_available():+ device = "cuda:0"+ elif torch.xpu.is_available():+ device = "xpu:0"++ W = linear.weight.data.to(device).clone() # TODO check is nesscessory+ if isinstance(linear, nn.Conv2d):+ W = W.flatten(1)+ if isinstance(linear, transformers.pytorch_utils.Conv1D):+ W = W.t()++ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(linear.weight, self.group_size)+ if self.is_nv:+ assert global_scale is not None and global_scale.numel() == 1+ scaled_tensor = tensor.to(global_scale.dtype) * get_reciprocal(+ scales.reshape(tensor.shape[0], -1) * get_reciprocal(global_scale)+ )+ scaled_tensor = cast_to_fp4(torch.clamp(scaled_tensor, -6.0, 6.0))+ else:+ scaled_tensor = tensor / (2 ** scales.reshape(tensor.shape[0], -1))+ scaled_tensor = revert_tensor_by_pad(scaled_tensor, orig_shape=orig_shape, pad_len=pad_len)+ if self.is_mx:+ final_scale = (scales + E8M0_EXPONENT_BIAS).clamp(0, E8M0_EXPONENT_NAN_VAL).to(torch.uint8)+ else:+ final_scale = scales.to(torch.float8_e4m3fn)+ self.weight_scale = final_scale+ # self.weight = get_compressed_weight(scaled_tensor, self.bits, self.data_type) ## TODO+ if self.bits == 8:+ compress_dtype = torch.float8_e4m3fn+ self.weight = scaled_tensor.to(compress_dtype)+ else:+ compress_dtype = torch.uint8+ self.weight_packed = self.pack_fp4_to_uint8(scaled_tensor)++ if global_scale is not None:+ self.weight_global_scale = global_scale.to(torch.float32).to(device)++ if input_global_scale is not None:+ self.input_global_scale = input_global_scale.to(torch.float32).to(device)+ return++ def pack_fp4_to_uint8(self, x: torch.Tensor) -> torch.Tensor:+ """+ Packs a tensor with values in the fp4 range into uint8.+ As there are 16 valid fp4 values, two fp4 values can be+ packed into one uint8. Each fp4 value is mapped to its+ particular index (e.g. 0.5 is mapped to index 1, 6.0 is mapped+ to index 7) which is then represented using 4 bits. Consecutive+ pairs of 4 bits are then packed into an uint8.++ :param x: tensor to pack+ returns: a packed tensor in uint8+ """++ m, n = x.shape+ device = x.device++ # Create lookup table for FP4 values to indices+ # Map the absolute values to 0-7 indices+ kE2M1 = torch.tensor(FLOAT_TO_E2M1, device=device, dtype=x.dtype)++ # Find closest valid FP4 value index for each element+ abs_x = torch.abs(x)+ abs_indices = torch.zeros_like(abs_x, dtype=torch.long)+ for i, val in enumerate(kE2M1): # TODO any optimize?+ abs_indices = torch.where(torch.isclose(abs_x, val), i, abs_indices)++ # Apply sign bit (bit 3) to get final 4-bit representation+ indices = abs_indices + (torch.signbit(x) << 3).to(torch.long)++ # Reshape to prepare for packing pairs of values+ indices = indices.reshape(-1)++ # Handle odd length by padding if necessary+ if indices.numel() % 2 != 0:+ indices = torch.cat([indices, torch.zeros(1, dtype=torch.long, device=device)])++ # Reshape to pair consecutive elements+ indices = indices.reshape(-1, 2)++ # Pack pairs of 4-bit values into 8-bit values+ packed = (indices[:, 0] | (indices[:, 1] << 4)).to(torch.uint8)++ return packed.reshape(m, n // 2)++ # def get_fp_scale(self, scale_e8m0):+ # E8M0_EXPONENT_BIAS = 127+ # E8M0_EXPONENT_NAN_VAL = 255++ # scale_e8m0 = scale_e8m0.view(torch.uint8)+ # s_offset = scale_e8m0.to(torch.int16) - E8M0_EXPONENT_BIAS+ # # TODO(later): it would be nice if there was a way to do the 2^x operation+ # # in PyTorch without creating a tensor of twos+ # two = torch.full(s_offset.size(), 2.0, device=scale_e8m0.device)+ # # pow(two, s_offset) can be out of range of floating point formats.+ # # TODO(later): handle this for float16 if we decide to support float16+ # # scales.+ # s_fp = torch.pow(two, s_offset)++ # # If a block exponent was 255, set values of that block to NaN+ # s_fp = torch.where(scale_e8m0 != E8M0_EXPONENT_NAN_VAL, s_fp, float("nan"))++ # return s_fp++ # def get_compressed_weight(self, tensor, bits, data_type):+ # data_type = str(data_type)+ # compress_dtype = None+ # if self.is_mx:+ # if bits == 4:++ # if self.is_nv:+ # pass ## TODOdiff --git a/auto_round/export/export_to_autoround/utils.py b/auto_round/export/export_to_autoround/utils.pynew file mode 100644index 000000000..11b217424--- /dev/null+++ b/auto_round/export/export_to_autoround/utils.py@@ -0,0 +1,37 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+++def check_neq_config(config, data_type, bits, act_bits, group_size, sym):+ """+ Checks if the provided configuration parameters are not equal to the values in the config dictionary.++ Args:+ config (dict): A dictionary containing the configuration parameters.+ data_type (str): The expected data type.+ bits (int): The expected number of bits.+ group_size (int): The expected group size.+ sym (bool): The expected symmetry flag.++ Returns:+ list: A list of strings indicating which configuration parameters do not match.+ """+ expected_config = {+ "data_type": data_type,+ "bits": bits,+ "group_size": group_size,+ "sym": sym,+ "act_bits": act_bits,+ }+ return [key for key, expected_value in expected_config.items() if config.get(key) != expected_value]diff --git a/auto_round/export/export_to_llmcompressor/__init__.py b/auto_round/export/export_to_llmcompressor/__init__.pyindex 0d2740cdb..1ab77d2e5 100644--- a/auto_round/export/export_to_llmcompressor/__init__.py+++ b/auto_round/export/export_to_llmcompressor/__init__.py@@ -11,3 +11,4 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.+from .config import check_compressed_tensors_supporteddiff --git a/auto_round/export/export_to_llmcompressor/config.py b/auto_round/export/export_to_llmcompressor/config.pyindex 6b3145c98..9b205123e 100644--- a/auto_round/export/export_to_llmcompressor/config.py+++ b/auto_round/export/export_to_llmcompressor/config.py@@ -52,3 +52,87 @@ quantization_config = {"quantization_status": "compressed","provider": "auto-round", # This is the key to indicate the model provider}++# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from auto_round.utils import logger+++def check_compressed_tensors_supported(): # pragma: no cover+ try:+ import compressed_tensors # noqa: F401++ return True+ except ImportError:+ logger.warning(+ "Please install compressed-tensors via 'pip install compressed-tensors'" " to save as llm-compressor format"+ )+ return False+++if check_compressed_tensors_supported():+ from compressed_tensors.quantization import ( # pylint: disable=E0401+ QuantizationConfig,+ QuantizationScheme,+ QuantizationStatus,+ is_preset_scheme,+ preset_name_to_scheme,+ )+++# please refer to https://github.com/vllm-project/llm-compressor/blob/+# 29f4d5644b48e9c8ebb7e36d5be9f7c92747ceb7/src/llmcompressor/modifiers/quantization/quantization/mixin.py#L168+def initialize_quantization(scheme, targets=["Linear"], config_groups=None, kv_cache_scheme=None, ignore=["lm_head"]):+ """+ Attach quantization schemes to modules in the model and initialize the quantization config+ """++ # apply scheme and status to model+ scheme = scheme+ targets = targets+ config_groups = config_groups+ kv_cache_scheme = kv_cache_scheme+ ignore = ignore+ check_compressed_tensors_supported()+ if scheme is not None and config_groups is not None:+ raise ValueError("Please specify either `scheme` or `config_groups`")++ if scheme is not None:+ # takes precedence over config_groups++ if isinstance(scheme, str) and is_preset_scheme(scheme):+ # attach targets to scheme+ scheme = {scheme: targets}++ config_groups = {}+ for idx, key in enumerate(scheme.keys()):+ if is_preset_scheme(key):+ scheme = preset_name_to_scheme(key, scheme[key])+ else:+ scheme = QuantizationScheme.model_validate({"targets": scheme[key], **scheme})++ group_name = f"group_{idx}"+ config_groups[group_name] = scheme++ if config_groups is None or len(config_groups) == 0:+ default_quant_scheme = QuantizationScheme(targets=targets)+ config_groups = {"group_0": default_quant_scheme}++ return QuantizationConfig(+ config_groups=config_groups,+ kv_cache_scheme=kv_cache_scheme,+ quantization_status=QuantizationStatus.COMPRESSED,+ ignore=ignore,+ )diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex e5863f374..0f759384b 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -17,9 +17,11 @@ import osimport torchfrom auto_round.export.export_to_llmcompressor.config import quantization_config-from auto_round.utils import detect_device, get_module, logger, set_module+from auto_round.utils import detect_device, get_module, is_mx_fp, is_nv_fp, is_standard_fp, logger, set_modulefrom auto_round.wrapper import WrapperWALayer+from .export_to_fp import save_quantized_as_fp+@torch.no_grad()def recover_qweight(qdq_weight, scale):@@ -36,8 +38,40 @@ def recover_qweight(qdq_weight, scale):return (qdq_weight / scale).to(torch.int8)+def pack_layer(layer_name, model, backend):+ """+ Packs a model layer for quantization based on its type and configuration.++ This function retrieves the specified layer from the model, checks its+ compatibility for quantization, and replaces it with a quantized version+ if applicable. The quantization process depends on the layer's bit-width,+ group size, symmetry, and activation bits.++ Args:+ layer_name (str): The name of the layer to be packed.+ model (torch.nn.Module): The model containing the layer.+ backend (str): The backend framework to be used for quantization.++ Returns:+ None: The function modifies the model in place.+ """+ if is_nv_fp(backend) or is_mx_fp(backend):+ from auto_round.export.export_to_llmcompressor.export_to_fp import pack_layer++ return pack_layer(layer_name, model, backend)++ ## passed as no other llmcompressor format is supported yet+ logger.warning("No other llmcompressor packing format(except NVFP&MXFP) is supported yet, skip packing")+ return++@torch.no_grad()-def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):+def save_quantized_as_llmcompressor(output_dir, **kwargs):+ backend = kwargs.get("backend", None)+ if is_nv_fp(backend) or is_mx_fp(backend):+ return save_quantized_as_fp(output_dir, **kwargs)++ model = kwargs.get("model", None)safe_serialization = kwargs.get("safe_serialization", True)tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pynew file mode 100644index 000000000..0b989f65d--- /dev/null+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -0,0 +1,278 @@+# Copyright (c) 2023 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++import copy+import inspect+import json+import os+from concurrent.futures import ThreadPoolExecutor++import threadpoolctl as tctl+import torch+import torch.nn as nn+import transformers+from tqdm import tqdm++from auto_round.export.export_to_autoround.qlinear_fp import QuantLinear+from auto_round.utils import (+ SUPPORTED_LAYER_TYPES,+ check_start_with_block_name,+ check_to_quantized,+ filter_quantization_config,+ get_block_names,+ get_module,+ is_mx_fp,+ is_nv_fp,+ logger,+ set_amax_for_all_moe_layers,+ set_module,+)+from auto_round.wrapper import WrapperWALayer++from .config import check_compressed_tensors_supported++__all__ = [+ "pack_layer",+ "save_quantized_as_fp",+]+++def pack_layer(name, model, backend):+ if name == "lm_head": # TODO: Check vLLM inference status to determine whether to enable this feature+ return+ layer = get_module(model, name)++ if not isinstance(layer, SUPPORTED_LAYER_TYPES) and not isinstance(layer, WrapperWALayer): ##already packed+ return++ if isinstance(layer, WrapperWALayer): # revert WrapperWALayer for offline usage+ wp_layer = layer+ layer = wp_layer.orig_layer+ set_module(model, name, layer)++ data_type = layer.data_type+ act_bits = layer.act_bits+ act_data_type = layer.act_data_type+ bits = layer.bits+ if bits > 8:+ return+ group_size = layer.group_size+ sym = layer.sym+ device = layer.weight.device++ if is_nv_fp(act_data_type) and act_bits <= 8:+ if not getattr(layer, "input_global_scale", None):+ assert hasattr(layer, "act_max")+ from auto_round.data_type.nvfp import calculate_gparam++ input_global_scale = calculate_gparam(layer.act_max, layer.group_size) # , model.device+ setattr(layer, "input_global_scale", input_global_scale)+ delattr(layer, "act_max")++ # QuantLinear = get_fp_qlinear(backend, bits, group_size, sym)++ if isinstance(layer, nn.Linear):+ in_features = layer.in_features+ out_features = layer.out_features+ elif isinstance(layer, nn.Conv2d):+ in_features = layer.in_channels+ out_features = layer.out_channels+ elif isinstance(layer, transformers.pytorch_utils.Conv1D):+ in_features = layer.weight.shape[0]+ out_features = layer.weight.shape[1]++ bias = layer.bias is not None+ ##bias = True ## if using the above, llama3 lambada RTN will be NAN , TODO why?+ new_layer = QuantLinear( ##pylint: disable=E1123+ bits,+ group_size,+ in_features,+ out_features,+ bias,+ weight_dtype=layer.weight.dtype,+ sym=sym,+ data_type=data_type,+ act_bits=act_bits,+ act_data_type=act_data_type,+ )++ new_layer.device = device+ set_module(model, name, new_layer)+ qlayer = new_layer+ scale = layer.scale+ global_scale = getattr(layer, "weight_global_scale", None)+ input_global_scale = getattr(layer, "input_global_scale", None)+ # zero = layer.zp+ # so far can only pack layer on CPU+ qlayer.to("cpu")+ layer, scale = layer.to("cpu"), scale.to("cpu")+ qlayer.pack(layer, scale, global_scale=global_scale, input_global_scale=input_global_scale)+ ## no zeros to handle, as mxfp not support asym quantization+ qlayer.to(device)+++def save_quantized_as_fp(output_dir, inplace=True, **kwargs):+ """+ Saves a quantized model of mxfp/nvfp data_type in the llm-compressor format.++ Args:+ output_dir (str): The directory where the quantized model will be saved.+ inplace (bool, optional): If True, modifies the model in place. Otherwise, creates a deepcopy of the model.+ Default is True.+ backend (str, optional): The backend to be used for quantization.+ Default is "autoround:exllamav2".+ **kwargs: Additional keyword arguments including:+ - model (nn.Module): The model to be quantized.+ - layer_config (dict): The layer configuration for each layer.+ - serialization_dict (dict): The serialization configuration.+ - tokenizer (Tokenizer, optional): The tokenizer to be saved.++ Returns:+ None++ Raises:+ ValueError: If the backend is not supported.+ """+ model = kwargs["model"]+ backend = kwargs.get("backend", None)+ bits = kwargs.get("bits", None)+ data_type = kwargs.get("data_type", None)+ act_bits = kwargs.get("act_bits", None)+ act_data_type = kwargs.get("act_data_type", None)+ safe_serialization = True if "safe_serialization" not in kwargs.keys() else kwargs["safe_serialization"]+ if not inplace:+ model = copy.deepcopy(model.to("cpu"))+ layer_config = kwargs["layer_config"]+ tokenizer = kwargs.get("tokenizer", None)+ processor = kwargs.get("processor", None)+ extra_config = {}++ if act_bits <= 8:+ # revert WrapperWALayer for offline usage+ for n, m in model.named_modules():+ if isinstance(m, WrapperWALayer):+ orig_layer = m.orig_layer+ if not getattr(orig_layer, "input_global_scale", None):+ assert hasattr(orig_layer, "act_max")+ from auto_round.data_type.nvfp import calculate_gparam++ input_global_scale = calculate_gparam(orig_layer.act_max, orig_layer.group_size, model.device)+ setattr(orig_layer, "input_global_scale", input_global_scale)+ delattr(orig_layer, "act_max")+ set_module(model, n, orig_layer)++ # update input_global_scale+ from auto_round.data_type.utils import update_fused_layer_global_scales++ modules = list(model.modules())+ for module in tqdm(modules, desc="Update input global scale for fuse modules"):+ update_fused_layer_global_scales(module, base_name="input")++ names = list(layer_config.keys())+ max_workers = 1+ if not torch.cuda.is_available() or not torch.xpu.is_available():+ max_workers = 2 ## 2 with cuda packing will cause hang occasionally+ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ with tqdm(total=len(names), leave=True) as pbar:++ def wrapper(name):+ pbar.set_description(f"packing {name}")+ with tctl.threadpool_limits(limits=1):+ pack_layer(name, model, backend)+ pbar.update(1)++ for _ in executor.map(wrapper, names):+ pass++ # TODO fix the ignore re match issue, compile with fp8 & int8 config+ ignore = ["lm_head"]+ for layer_name in layer_config:+ if layer_config[layer_name]["bits"] > 8: ## find ignore layers+ ignore.append(layer_name)+ ignore = list(set(ignore))++ # get llm-compressor format config+ check_compressed_tensors_supported()+ from .config import initialize_quantization++ scheme = "NVFP4"+ quantization_config = initialize_quantization(scheme=scheme)+ setattr(quantization_config, "format", "nvfp4-pack-quantized")+ setattr(quantization_config, "ignore", ignore)+ quantization_config = quantization_config.to_dict()+ if is_mx_fp(data_type): # Manually replace some parameters, as compressed-tensor is not support MXFP scheme yet+ quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] = bits+ quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] = (+ act_bits if act_bits <= 8 else bits+ )+ quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] = True+ quantization_config["config_groups"]["group_0"]["input_activations"]["is_mx"] = True+ quantization_config["config_groups"]["group_0"]["weights"]["group_size"] = 32+ quantization_config["config_groups"]["group_0"]["input_activations"]["group_size"] = 32+ quantization_config["format"] = "float-quantized"+ if hasattr(model, "config"):+ model.config.quantization_config = quantization_config+ if output_dir is None:+ return model++ if output_dir is None:+ model.tokenizer = tokenizer+ return model+ if os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")+ if tokenizer is not None:+ tokenizer.save_pretrained(output_dir)++ if processor is not None:+ processor.save_pretrained(output_dir)++ dtype = None+ save(model, output_dir, safe_serialization=safe_serialization, dtype=dtype)++ return model+++def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_serialization: bool = True, dtype=None):+ """Save model state dict and configs.++ Args:+ model (`nn.Module`):+ Model to be saved. The model can be wrapped or unwrapped.+ save_dir (`str`):+ Directory to which to save. Will be created if it doesn't exist.+ max_shard_size (`str`, defaults to `"10GB"`):+ The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size+ lower than this size. If expressed as a string, needs to be digits followed by a unit (like `"5MB"`).+ <Tip warning={true}>++ If a single weight of the model is bigger than `max_shard_size`, it will be in its own checkpoint shard+ which will be bigger than `max_shard_size`.++ </Tip>+ safe_serialization (`bool`, defaults to `True`):+ Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).+ """+ os.makedirs(save_dir, exist_ok=True)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ config_path = os.path.join(save_dir, "config.json")+ if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):+ with open(config_path, "r") as file:+ data = json.load(file)+ data["torch_dtype"] = str(dtype).split(".")[-1]+ with open(config_path, "w") as file:+ json.dump(data, file, indent=2)+ config_file = "quantization_config.json"+ if hasattr(model, "config") and hasattr(model.config, "quantization_config"):+ with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:+ json.dump(model.config.quantization_config, f, indent=2)diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 9b83aa371..08b693c26 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -483,7 +483,7 @@ def tune(args):layer_config = {}not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)for name in not_quantize_layer_names:- layer_config[name] = {"bits": 16}+ layer_config[name] = {"bits": 16, "act_bits": 16}if len(not_quantize_layer_names) > 0:logger.info(f"{not_quantize_layer_names} will not be quantized.")for format in formats:@@ -508,7 +508,7 @@ def tune(args):breakif args.quant_lm_head:- layer_config[lm_head_layer_name] = {"bits": args.bits}+ layer_config[lm_head_layer_name] = {"bits": args.bits, "act_bits": args.act_bits}for format in formats:if "auto_round" not in format and "fake" not in format:auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 86c850628..1f55402df 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -369,7 +369,7 @@ def tune(args):layer_config = {}not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)for name in not_quantize_layer_names:- layer_config[name] = {"bits": 16}+ layer_config[name] = {"bits": 16, "act_bits": 16}if len(not_quantize_layer_names) > 0:logger.info(f"{not_quantize_layer_names} will not be quantized.")for format in formats:@@ -408,7 +408,7 @@ def tune(args):)breakif args.quant_lm_head:- layer_config[lm_head_layer_name] = {"bits": args.bits}+ layer_config[lm_head_layer_name] = {"bits": args.bits, "act_bits": args.act_bits}for format in formats:if "auto_round" not in format and "fake" not in format:auto_round_formats = [s for s in SUPPORTED_FORMATS if s.startswith("auto_round")]diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 8b8530cb4..2b8ef540e 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -12,14 +12,17 @@# See the License for the specific language governing permissions and# limitations under the License.+import collections.abcimport copyimport gc+import jsonimport loggingimport osimport reimport subprocessimport sysfrom collections import UserDict+from enum import Enumfrom functools import lru_cacheimport cpuinfo@@ -48,6 +51,7 @@ class SupportedFormats:"itrex","itrex_xpu","fake",+ "llmcompressor",)self._gguf_format = tuple(sorted(GGUF_CONFIG.keys()))self._support_list = self._support_format + self._gguf_format@@ -1449,7 +1453,6 @@ def mllm_load_model(model_dtype=None,**kwargs,):- import jsonimport transformersfrom huggingface_hub import HfApi, HfFileSystem, hf_hub_download@@ -2249,3 +2252,167 @@ def download_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):model_path = snapshot_download(repo_id)return model_path+++def is_moe(module: torch.nn.Module) -> bool:+ """Returns whether the module is an MOE layer."""+ return any(+ key in type(module).__name__.lower()+ for key in [+ "MixtralSparseMoeBlock".lower(),+ "ArcticMoE".lower(),+ "DbrxFFN".lower(),+ "MoELayer".lower(),+ "PhimoeSparseMoeBlock".lower(),+ "DeepseekMoE".lower(),+ "DeepseekV2MoE".lower(),+ "DeepseekV3MoE".lower(),+ "Qwen2MoeSparseMoeBlock".lower(),+ "Qwen3MoeSparseMoeBlock".lower(),+ ]+ )+++# please refer to https://github.com/NVIDIA/TensorRT-Model-Optimizer+# /blob/4c611e47a60084a86e1de7e48690a692a1b8170c/modelopt/torch/export/layer_utils.py#L976+def get_expert_linear_names(module: torch.nn.Module) -> list[str]:+ """Get the list of linear names for the experts."""++ def module_match_name_list(module, name_list):+ """Check if the module name matches any of the names in the list.++ e.g. module_match_name_list(QuantQwen3MoeSparseMoeBlock, ['Qwen3MoeSparseMoeBlock']) -> True++ """+ return any(name.lower() in type(module).__name__.lower() for name in name_list)++ if module_match_name_list(+ module, ["Qwen2MoeSparseMoeBlock", "Qwen3MoeSparseMoeBlock", "DeepseekMoE", "DeepseekV2MoE", "DeepseekV3MoE"]+ ):+ return ["gate_proj", "down_proj", "up_proj"]+ elif module_match_name_list(module, ["MixtralMoeSparseMoeBlock"]):+ return ["linear_fc1", "linear_fc2"]+ elif module_match_name_list(module, ["DBRXMoeSparseMoeBlock"]):+ return ["w1_linear", "w2_linear", "v1_linear"]+ else:+ # assuing w1, w2, w3 by default+ return ["w1", "w2", "w3"]+++def get_nested_attr(module, attr_name: str):+ """Recursively get nested attribute (e.g., 'orig_layer.act_max')."""+ attrs = attr_name.split(".")+ for attr in attrs:+ if not hasattr(module, attr):+ return None+ module = getattr(module, attr)+ return module+++def set_nested_attr(module, attr_name: str, value):+ """Recursively set nested attribute (e.g., 'orig_layer.act_max' = value)."""+ attrs = attr_name.split(".")+ for attr in attrs[:-1]:+ if not hasattr(module, attr):+ raise AttributeError(f"{module} has no attribute '{attr}'")+ module = getattr(module, attr)+ setattr(module, attrs[-1], value)+++def set_amax_for_uncalibrated_experts(+ experts: torch.nn.Module, set_amax_value: float | None = None, attr_name="act_max"+):+ """Set amax of uncalibrated experts to a given value or the max of existing amax value from other experts.++ Args:+ experts: a list of experts+ set_amax_value: set amax value to the given value.+ If None, set amax value to the max of existing amax value from other experts.++ Returns:+ uncalibrated_experts: a list of uncalibrated experts+ """+ uncalibrated_experts = []+ # get the max amax value from all experts+ if set_amax_value is None:+ amax_values = [+ get_nested_attr(module, attr_name) for module in experts if get_nested_attr(module, attr_name) is not None+ ]+ if len(amax_values) == 0:+ return uncalibrated_experts+ # Flatten all tensors to 1D before concatenation+ flat_values = [t.reshape(-1) for t in amax_values]+ all_values = torch.cat(flat_values)+ set_amax_value = torch.max(all_values)++ for module in experts:+ if get_nested_attr(module, attr_name) is None:+ logger.warning_once(+ "Missing amax value of expert layers."+ "This typically occurs in MoE models when certain experts are not activated during calibration. "+ "Consider increasing your calibration dataset size to ensure all experts are exercised."+ )+ # Use float32 dtype explicitly to ensure we create a floating point tensor+ if not isinstance(set_amax_value, torch.Tensor):+ set_amax_value = torch.tensor(set_amax_value, dtype=torch.float32)+ set_nested_attr(module, attr_name, set_amax_value)+ # uncalibrated_experts.append(module)+++# Please refer to: https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/+# 4c611e47a60084a86e1de7e48690a692a1b8170c/modelopt/torch/export/unified_export_hf.py#L195-L207+def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_name="act_max"):+ if layer_name is not None:+ parts = layer_name.split(".")+ if "experts" not in parts:+ raise ValueError+ idx = parts.index("experts")+ moe_name = ".".join(parts[:idx])+ model = get_module(model, moe_name)+ # Handle input quantizers of experts that are not calibrated+ for name, sub_module in model.named_modules():+ if is_moe(sub_module) and hasattr(sub_module, "experts"):+ expert_linear_names = get_expert_linear_names(sub_module)+ for linear_name in expert_linear_names:+ if isinstance(sub_module.experts, collections.abc.Iterable):+ # For other MoE models (like Mixtral) with iterable experts+ try:+ set_amax_for_uncalibrated_experts(+ [getattr(expert, linear_name) for expert in sub_module.experts], attr_name=attr_name+ )+ except AttributeError as e:+ # Provide more helpful debugging information+ expert_types = list(set(type(expert).__name__ for expert in sub_module.experts))+ raise AttributeError(+ f"Failed to access attribute '{linear_name}' on experts. "+ f"MoE module type: {type(sub_module).__name__}, "+ f"Expert types: {expert_types}, "+ f"Expected linear names: {expert_linear_names}. "+ f"This suggests the get_expert_linear_names function may need "+ f"to be updated for this model architecture. "+ f"Original error: {e}"+ ) from e+ else:+ # Unsupported MoE model structure+ raise NotImplementedError(+ f"MoE model with experts type '{type(sub_module.experts).__name__}' is not supported in export."+ f"Please file an issue or add support for this model architecture."+ )+++class BackendDataType(str, Enum):+ STANDARD_FP8 = "fp8"+ MX_FP = "mx_fp"+ NV_FP = "nv_fp"+++def is_standard_fp(backend):+ return BackendDataType.STANDARD_FP8 in backend and not is_mx_fp(backend) and not is_nv_fp(backend)+++def is_mx_fp(backend):+ return BackendDataType.MX_FP in backend+++def is_nv_fp(backend):+ return BackendDataType.NV_FP in backenddiff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex be3427424..591e5c567 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -18,7 +18,7 @@ from torch.functional import Ffrom auto_round.data_type import get_quant_func-from .utils import SUPPORTED_LAYER_TYPES, check_to_quantized, get_scale_shape, logger, set_module+from .utils import SUPPORTED_LAYER_TYPES, check_to_quantized, get_scale_shape, is_mx_fp, is_nv_fp, logger, set_moduledef reshape_and_pad_tensor(v, group_size=-1):@@ -82,6 +82,13 @@ class WrapperLinear(torch.nn.Module):self.enable_round_tuning = enable_round_tuningself.enable_norm_bias_tuning = enable_norm_bias_tuning and (orig_layer.bias is not None)self.enable_act_quant = self.orig_layer.act_bits <= 8+ self.weight_global_scale = getattr(self.orig_layer, "weight_global_scale", None)+ if is_nv_fp(self.orig_layer.data_type) and self.weight_global_scale is None:+ from auto_round.data_type.nvfp import calculate_gparam++ weight_global_scale = calculate_gparam(self.orig_layer.weight, self.orig_layer.group_size)+ setattr(self, "weight_global_scale", weight_global_scale)+ self.weight_global_scale = self.weight_global_scale.to(self.orig_layer.weight.device)if hasattr(self.orig_layer, "scale_dtype") and self.orig_layer.scale_dtype == torch.float32:self.q_scale_thresh = 1e-8else:@@ -185,6 +192,7 @@ class WrapperLinear(torch.nn.Module):data_type=self.data_type,q_scale_thresh=self.q_scale_thresh,imatrix=self.orig_layer.imatrix if hasattr(self.orig_layer, "imatrix") else None,+ global_scale=getattr(self, "weight_global_scale", None),**quant_kwargs,)weight_q = weight_q.to(weight.dtype)@@ -213,6 +221,7 @@ class WrapperLinear(torch.nn.Module):data_type=self.act_data_type,max_scale=act_max_scale,tensor_max=act_max,+ global_scale=getattr(self, "weight_global_scale", None),)return x, scale, zp@@ -229,7 +238,12 @@ class WrapperLinear(torch.nn.Module):bias_bits = 4 ## hard codebias_group_size = -1bias, scale, zp = self.bias_quant_func(- bias, bits=bias_bits, group_size=bias_group_size, v=bias_v, q_scale_thresh=self.q_scale_thresh+ bias,+ bits=bias_bits,+ group_size=bias_group_size,+ v=bias_v,+ q_scale_thresh=self.q_scale_thresh,+ global_scale=getattr(self, "weight_global_scale", None),)return bias, scale, zp@@ -280,8 +294,6 @@ class WrapperLinear(torch.nn.Module):if zp is not None:if isinstance(zp, dict):_set_dict_attr(zp, "zp")- elif zp is None:- self.orig_layer.zp = Noneelif zp.numel() > 1:zp = zp.reshape(shape[0], -1)self.orig_layer.zp = zp.to("cpu")@@ -290,6 +302,11 @@ class WrapperLinear(torch.nn.Module):else:self.orig_layer.zp = None+ if self.weight_global_scale is not None:+ global_scale = self.weight_global_scale+ assert global_scale.numel() == 1+ self.orig_layer.weight_global_scale = global_scale.to("cpu")+##unwrapper biasif self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quantbias_v = best_params["bias_v"].to(self.device)@@ -373,7 +390,7 @@ class WrapperLinear(torch.nn.Module):torch.Tensor: Output tensor after applying the wrapped layer."""x = x.to(self.device)- weight_q, _, _ = self._qdq_weight(self.value, self.min_scale, self.max_scale)+ weight_q, *_ = self._qdq_weight(self.value, self.min_scale, self.max_scale)if self.enable_act_quant:act_max = self.orig_layer.act_max if hasattr(self.orig_layer, "act_max") else None@@ -584,7 +601,6 @@ def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device="set_module(block, n, new_m)else:logger.warning_once(f"{m.__class__.__name__} is not supported")-return quantized_layers, unquantized_layersdiff --git a/test/test_cpu/requirements.txt b/test/test_cpu/requirements.txtindex bc6615906..265ba7e94 100644--- a/test/test_cpu/requirements.txt+++ b/test/test_cpu/requirements.txt@@ -2,3 +2,4 @@ addictmodelscopegguftorchvision+compressed-tensorsdiff --git a/test/test_cpu/test_export.py b/test/test_cpu/test_export.pyindex bbce4036b..b60c87f26 100644--- a/test/test_cpu/test_export.py+++ b/test/test_cpu/test_export.py@@ -252,6 +252,169 @@ class TestAutoRound(unittest.TestCase):self.assertEqual(f.get_tensor("model.decoder.layers.5.self_attn.v_proj.weight").dtype, torch.float8_e4m3fn)shutil.rmtree(quantized_model_path, ignore_errors=True)+ def test_mxfp4_llmcompressor_format(self):+ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ from transformers import AutoConfig++ bits = 4+ data_type = "mx_fp4e2m1"+ group_size = 32+ sym = True+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ data_type=data_type,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ autoround.quantize()+ compressed_model = autoround.save_quantized(+ output_dir=quantized_model_path, inplace=True, format="llmcompressor"+ )+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_packed")+ and tmp_layer.weight_scale.dtype is torch.uint8+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal MXFP4 packing name or data_type or shape"+ quantization_config = AutoConfig.from_pretrained(+ quantized_model_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "float-quantized"+ and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4+ ), f"Invalid MXFP4 quantization configuration: {quantization_config}"+ shutil.rmtree("./saved", ignore_errors=True)++ def test_mxfp8_llmcompressor_format(self):+ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ from transformers import AutoConfig++ bits = 8+ data_type = "mx_fp8e4m3_rceil"+ group_size = 32+ sym = True+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ data_type=data_type,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llmcompressor")+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight")+ and tmp_layer.weight.dtype is torch.float8_e4m3fn+ and tmp_layer.weight_scale.dtype is torch.uint8+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal MXFP8 packing name or data_type or shape"+ quantization_config = AutoConfig.from_pretrained(+ quantized_model_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "float-quantized"+ and quantization_config["config_groups"]["group_0"]["weights"]["is_mx"] is True+ and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8+ ), f"Invalid MXFP8 quantization configuration: {quantization_config}"+ shutil.rmtree("./saved", ignore_errors=True)++ def test_nvfp4_llmcompressor_format(self):+ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ from transformers import AutoConfig++ bits = 4+ act_bits = 4+ data_type = "nv_fp"+ act_data_type = "nv_fp4_with_static_gs"+ group_size = 16+ sym = True+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=bits,+ act_bits=act_bits,+ data_type=data_type,+ act_data_type=act_data_type,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llmcompressor")+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_global_scale")+ and hasattr(tmp_layer, "input_global_scale")+ and tmp_layer.weight_packed.dtype is torch.uint8+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal NVFP4 packing name or data_type or shape"+ quantization_config = AutoConfig.from_pretrained(+ quantized_model_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "nvfp4-pack-quantized"+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 4+ ), f"Invalid NVFP4 quantization configuration: {quantization_config}"+ shutil.rmtree("./saved", ignore_errors=True)++ def test_nvfp4_autoround_format(self):+ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ from transformers import AutoConfig++ bits = 4+ act_bits = 4+ data_type = "nv_fp"+ act_data_type = "nv_fp4_with_static_gs"+ group_size = 16+ sym = True+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=bits,+ act_bits=act_bits,+ data_type=data_type,+ act_data_type=act_data_type,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_global_scale")+ and hasattr(tmp_layer, "input_global_scale")+ and tmp_layer.weight_packed.dtype is torch.uint8+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal NVFP4 packing name or data_type or shape"+ shutil.rmtree("./saved", ignore_errors=True)+if __name__ == "__main__":unittest.main()diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex 028a6fedd..9e3fc4b26 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -59,7 +59,7 @@ class TestGGUF(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")- gguf_file = os.listdir(quantized_model_path)[0]+ gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_0.gguf"model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"inputs = self.tokenizer(text, return_tensors="pt").to(model.device)@@ -79,7 +79,7 @@ class TestGGUF(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")- gguf_file = os.listdir(quantized_model_path)[0]+ gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_1.gguf"model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"inputs = self.tokenizer(text, return_tensors="pt").to(model.device)@@ -317,9 +317,13 @@ class TestGGUF(unittest.TestCase):quantized_model_path = "./saved"autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_0")self.assertTrue("mmproj-model.gguf" in os.listdir("./saved"))- file_name = os.listdir(quantized_model_path)[0]- file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2- self.assertAlmostEqual(file_size, 892, delta=1.0)+ for file_name in os.listdir(quantized_model_path):+ file_name = os.listdir(quantized_model_path)[0]+ file_size = os.path.getsize(os.path.join(quantized_model_path, file_name)) / 1024**2+ if file_name == "mmproj-model.gguf":+ self.assertAlmostEqual(file_size, 2535, delta=1.0)+ else:+ self.assertAlmostEqual(file_size, 892, delta=1.0)shutil.rmtree("./saved", ignore_errors=True)diff --git a/test/test_cuda/requirements.txt b/test/test_cuda/requirements.txtindex 1d9a5cb65..88b031217 100644--- a/test/test_cuda/requirements.txt+++ b/test/test_cuda/requirements.txt@@ -18,4 +18,6 @@ torchvisiontqdmtransformersvllm>=0.8.5.post1+compressed-tensors+diff --git a/test/test_cuda/test_export.py b/test/test_cuda/test_export.pyindex 36021d3d5..9e4711915 100644--- a/test/test_cuda/test_export.py+++ b/test/test_cuda/test_export.py@@ -6,7 +6,7 @@ import unittestsys.path.insert(0, "../..")import torchimport transformers-from transformers import AutoModelForCausalLM, AutoTokenizer+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom auto_round.testing_utils import require_awq, require_optimum@@ -25,7 +25,7 @@ class TestAutoRound(unittest.TestCase):@classmethoddef setUpClass(self):self.model_name = "facebook/opt-125m"-+ self.save_dir = "./saved"self.llm_dataloader = LLMDataLoader()@classmethod@@ -320,6 +320,71 @@ class TestAutoRound(unittest.TestCase):print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))shutil.rmtree("./saved", ignore_errors=True)+ def test_nvfp4_llmcompressor_format(self):+ model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)++ bits = 4+ act_bits = 4+ data_type = "nv_fp"+ act_data_type = "nv_fp4_with_static_gs"+ group_size = 16+ sym = True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ act_bits=act_bits,+ data_type=data_type,+ act_data_type=act_data_type,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ quantized_model_path = self.save_dir+ compressed_model, _ = autoround.quantize_and_save(output_dir=quantized_model_path, format="llmcompressor")+ tmp_layer = compressed_model.model.decoder.layers[3].self_attn.q_proj+ assert (+ hasattr(tmp_layer, "weight_scale")+ and hasattr(tmp_layer, "weight_global_scale")+ and hasattr(tmp_layer, "input_global_scale")+ and tmp_layer.weight_packed.dtype is torch.uint8+ and tmp_layer.weight_scale.dtype is torch.float8_e4m3fn+ and tmp_layer.weight_scale.shape[0] == 768+ ), "Illegal NVFP4 packing name or data_type or shape"+ quantization_config = AutoConfig.from_pretrained(+ quantized_model_path, trust_remote_code=True+ ).quantization_config+ assert (+ quantization_config["format"] == "nvfp4-pack-quantized"+ and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 4+ ), f"Invalid NVFP4 quantization configuration: {quantization_config}"+ shutil.rmtree("./saved", ignore_errors=True)+ # from vllm import LLM, SamplingParams+ # prompts = [+ # "The capital of France is",+ # "The future of AI is",+ # ]+ ## Create a sampling params object.+ # sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=20)+ # QUANTIZATION = "compressed-tensors"+ # llm = LLM(model=quantized_model_path,+ # # quantization=QUANTIZATION,+ # trust_remote_code=True,+ # tensor_parallel_size=1,+ # enforce_eager=True,+ # gpu_memory_utilization=0.7,+ # )+ # outputs = llm.generate(prompts, sampling_params)+ # # Print the outputs.+ # for output in outputs:+ # prompt = output.prompt+ # generated_text = output.outputs[0].text+ # if "France" in prompt:+ # assert "Paris" in generated_text+if __name__ == "__main__":unittest.main()