PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add FP8 dtype info to quant config (#1228)
Yi Liuafea815efe69
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex c03403d3b..a162a0cf9 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -35,6 +35,7 @@ from auto_round.utils import (copy_python_files_from_model_cache,get_module,get_packing_device,+ is_gaudi2,logger,set_module,unsupported_meta_device,@@ -145,6 +146,20 @@ def _use_fp8_kv(static_kv_dtype: str | None) -> bool:return False+def _configure_gaudi2_fp8_dtype(quantization_config: dict) -> None:+ """Configure FP8 dtype flavor for Intel Gaudi2 hardware compatibility."""+ _GAUDI2_FP8_DTYPE_FLAVOR = "float8_e4m3fnuz"++ if is_gaudi2():+ quantization_config["fp8_dtype_flavor"] = _GAUDI2_FP8_DTYPE_FLAVOR+ logger.warning_once(+ (+ "Running on Intel Gaudi2 hardware."+ f" Setting FP8 dtype flavor to {_GAUDI2_FP8_DTYPE_FLAVOR} for compatibility."+ )+ )++def save_quantized_as_static_fp(output_dir: str,model: torch.nn.Module = None,@@ -258,6 +273,7 @@ def save_quantized_as_static_fp()quantization_config = quantization_config.to_dict()quantization_config["provider"] = "auto-round"+ _configure_gaudi2_fp8_dtype(quantization_config)quantization_config["format"] = "float-quantized"if group_size > 0:quantization_config["config_groups"]["group_0"]["weights"]["group_size"] = group_size