PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Add G2-specific `FP8_STATIC` support (#1148)
Yi Liu547d43f810fa
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 85736d444..3e5e57069 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -656,8 +656,8 @@ class BaseCompressor(object):"'enable_torch_compile' is set to `False` by default. ""Enabling it can reduce tuning cost by 20%, but it might throw an exception.",)-- if (self.data_type.startswith("fp") or self.act_data_type.startswith("fp")) and self.enable_torch_compile:+ # On HPU, we rely on torch.compile to speed up the model execution.+ if self.enable_torch_compile and is_wfp8afp8(self) and not is_hpex_available():self.enable_torch_compile = Falselogger.warning("reset enable_torch_compile to `False` as fp8 is enabled")@@ -1927,7 +1927,6 @@ class BaseCompressor(object):Returns:The output tensor of the block."""-output = []nsamples = len(input_ids)for i in range(0, nsamples, bs):diff --git a/auto_round/data_type/fp8.py b/auto_round/data_type/fp8.pyindex 0fbd20020..fe108c3af 100644--- a/auto_round/data_type/fp8.py+++ b/auto_round/data_type/fp8.py@@ -22,6 +22,7 @@ from auto_round.data_type.utils import (reshape_pad_tensor_by_group_size,revert_tensor_by_pad,)+from auto_round.utils import is_gaudi2, logger@register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))@@ -220,3 +221,55 @@ def quant_fp8_sym_gaudi3(tensor, max_scale=1.0, tensor_max=None, **kwargs):qdq_res = fp8_res * scaleqdq_res = qdq_res.to(orig_dtype).reshape(orig_shape)return qdq_res, scale, None+++if is_gaudi2():++ @register_dtype(("fp8_sym", "fp8", "fp8_e4m3"))+ def quant_fp8_sym(tensor, max_scale=1.0, tensor_max=None, group_size=-1, v=0, **kwargs): # pylint: disable=E0102+ """Symmetric quantization using float8 format.++ Allows both dynamic per-token scaling and tensor-wide quantization depending on input.++ Args:+ tensor (torch.Tensor): Input tensor to quantize.+ max_scale (float, optional): Maximum scaling factor. Defaults to 1.0.+ tensor_max (float, optional): Maximum tensor value for precomputed scale. Defaults to None.+ **kwargs: Additional arguments for compatibility.++ Returns:+ tuple:+ - Quantized and dequantized tensor (torch.Tensor).+ - Scale tensor used for quantization (torch.Tensor).+ - Placeholder for zp (None).+ """+ logger.warning_once("Using float8_e4m3fnuz G2.")+ # !!! USE float8_e4m3fnuz for Gaudi2+ info = torch.finfo(torch.float8_e4m3fnuz)+ orig_dtype = tensor.dtype+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ if isinstance(max_scale, torch.Tensor):+ max_scale = max_scale.to(tensor.device)+ if isinstance(v, torch.Tensor):+ v = v.to(tensor.device)+ if tensor_max is None: ##dynamic per-token+ max_tensor = torch.max(torch.abs(tensor), dim=-1)[0] * max_scale+ elif isinstance(tensor_max, torch.Tensor):+ max_tensor = tensor_max.to(tensor.device) * max_scale+ else:+ max_tensor = torch.tensor(tensor_max).to(tensor.device) * max_scale+ scale = max_tensor.to(torch.float32) / info.max+ min_scaling_factor = float(1.0 / (info.max * 512.0)) ##copy from vllm+ scale = torch.clip(scale, min=min_scaling_factor)+ if tensor.dtype == torch.float16: ## Avoid NaN gradients with float16+ tensor = tensor.to(torch.bfloat16)+ scale = scale.unsqueeze(dim=-1)+ fp8_res = tensor / scale + v+ fp8_res = torch.clip(fp8_res, info.min, info.max)+ from auto_round.data_type.utils import float8_e4m3fnuz_hpu_ste as ste_fn++ fp8_res2 = ste_fn(fp8_res)+ qdq_res = fp8_res2 * scale+ qdq_res = revert_tensor_by_pad(qdq_res, orig_shape=orig_shape, pad_len=pad_len)+ qdq_res = qdq_res.to(orig_dtype)+ return qdq_res, scale, Nonediff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex a16830f36..890329f5c 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -220,6 +220,22 @@ def float8_e4m3fn_hpu_ste(x: torch.Tensor):return fp8+def float8_e4m3fnuz_hpu_ste(x: torch.Tensor):+ """Straight-Through Estimator (STE) for float8.++ Applies a quantization and dequantization step with float8 precision while maintaining+ gradient flow using a straight-through estimator.++ Args:+ x (torch.Tensor): Input tensor.++ Returns:+ torch.Tensor: Quantized and dequantized tensor using float8 format.+ """+ fp8 = ((torch.ops.hpu.cast_to_fp8_v2(x, 1.0, False, False, torch.float8_e4m3fn)[0]).to(x.dtype) - x).detach() + x+ return fp8++@lru_cache(None)def get_gaudi_fp8_ste_func():from auto_round.utils import is_hpex_availablediff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.pyindex d9d0ec212..bf813e967 100644--- a/auto_round/experimental/attention.py+++ b/auto_round/experimental/attention.py@@ -31,6 +31,7 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONSfrom auto_round.experimental.kv_cache import kvcache_quant_contextfrom auto_round.experimental.utils import (+ clean_model_parameters_and_buffers_,is_attention_module,per_tensor_fp8_qdq,update_parameter_data,@@ -146,6 +147,7 @@ def prep_attention_module_for_calibration(module: torch.nn.Module, config):def clean_up_hooked_attention(module, model):if is_attention_module(module):+ clean_model_parameters_and_buffers_(module, (QUERY_MAX_NAME,))# Cleanup phase: Restore the original attention implementationif hasattr(model.config, "_attn_implementation") and hasattr(model, "_original_impl"):model.config._attn_implementation = model._original_impldiff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.pyindex a26e384d9..545ebd401 100644--- a/auto_round/experimental/utils.py+++ b/auto_round/experimental/utils.py@@ -26,7 +26,7 @@ def per_tensor_fp8_qdq(return qdq_tensor, scale-# @torch.compiler.disabledef update_parameter_data(module: torch.nn.Module, new_val: torch.Tensor, name: str):"""Update the data of a parameter in a module.@@ -73,3 +73,29 @@ def is_attention_module(module: torch.nn.Module):return "attention" in module.__class__.__name__.lower() and (hasattr(module, "k_proj") or hasattr(module, "v_proj") or hasattr(module, "qkv_proj"))+++def _clean_param_or_buff_if_exists(module: torch.nn.Module, name_tuple: tuple[str, ...]):+ """+ Deletes parameters or buffers from a module if they exist.++ :param module: module to delete parameters/buffers from+ :param name_tuple: tuple of parameter/buffer names to delete+ """+ for name in name_tuple:+ if hasattr(module, name):+ try:+ delattr(module, name)+ except Exception as e:+ logger.warning(f"Could not delete {name} from module {module}: {e}")+++def clean_model_parameters_and_buffers_(model: torch.nn.Module, name_tuple: tuple[str, ...]):+ """+ Cleans parameters and buffers from all modules in the model.++ :param model: model to clean parameters/buffers from+ :param name_tuple: tuple of parameter/buffer names to delete+ """+ for module in model.modules():+ _clean_param_or_buff_if_exists(module, name_tuple)diff --git a/auto_round/utils/device.py b/auto_round/utils/device.pyindex 6090eeeb2..e19fbca68 100644--- a/auto_round/utils/device.py+++ b/auto_round/utils/device.py@@ -1336,6 +1336,17 @@ def parse_available_devices(device_map: Union[str, torch.device, int, dict, Noneraise TypeError(f"Unsupported device_map type: {type(device_map)}")+@lru_cache(maxsize=None)+def is_gaudi2():+ try:+ import habana_frameworks.torch.utils.experimental as htexp++ is_hpu_gaudi2 = htexp._get_device_type() == htexp.synDeviceType.synDeviceGaudi2+ return is_hpu_gaudi2+ except ImportError:+ return False++class MemoryMonitor:"""Global memory monitor for tracking peak RAM and VRAM usage."""diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex c9a08479c..f4bb15575 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -299,7 +299,6 @@ def llm_load_model(model = model_cls.from_pretrained(pretrained_model_name_or_path,torch_dtype=torch_dtype,- attn_implementation="eager",trust_remote_code=trust_remote_code,device_map="auto" if use_auto_mapping else None,)