PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: add per-head fp8 kv attention calibration (#2044)
Zeel Desaif17d9cd4b369
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex c4c58c318..3aff39ad9 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -336,6 +336,8 @@ _ENTRY_KWARG_OWNERS = {"enable_deterministic_algorithms": "base","static_kv_dtype": "base","static_attention_dtype": "base",+ "static_kv_granularity": "base",+ "static_attention_granularity": "base","processor": "mllm","image_processor": "mllm","template": "mllm",diff --git a/auto_round/cli/main.py b/auto_round/cli/main.pyindex e99595119..09b0b8263 100644--- a/auto_round/cli/main.py+++ b/auto_round/cli/main.py@@ -66,6 +66,10 @@ def _build_entry_base_kwargs(args, *, low_cpu_mem_usage, enable_torch_compile, l"layer_config": layer_config,"model_dtype": args.model_dtype,"trust_remote_code": not args.disable_trust_remote_code,+ "static_kv_dtype": args.static_kv_dtype,+ "static_kv_granularity": args.static_kv_granularity,+ "static_attention_dtype": args.static_attention_dtype,+ "static_attention_granularity": args.static_attention_granularity,}diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.pyindex 31001eab1..81ae5d3eb 100644--- a/auto_round/cli/parser.py+++ b/auto_round/cli/parser.py@@ -184,6 +184,13 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Arguchoices=["fp8", "float8_e4m3fn"],help="Static KV-cache quantization data type.",)+ rt.add_argument(+ "--static_kv_granularity",+ default="tensor",+ type=str,+ choices=["tensor", "head"],+ help="Static KV-cache FP8 calibration granularity.",+ )rt.add_argument("--static_attention_dtype",default=None,@@ -191,6 +198,13 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Arguchoices=["fp8", "float8_e4m3fn"],help="Static attention quantization data type.",)+ rt.add_argument(+ "--static_attention_granularity",+ default="tensor",+ type=str,+ choices=["tensor", "head"],+ help="Static attention FP8 calibration granularity.",+ )# ---- Evaluation ----ev = parser.add_argument_group("Evaluation Arguments")diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 8f6f3efd3..1943c0e5c 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -110,6 +110,8 @@ class SerializedCompressorConfig:supported_types: Optional[list[str]] = SUPPORTED_LAYER_TYPESstatic_attention_dtype: Optional[str] = Nonestatic_kv_dtype: Optional[str] = None+ static_attention_granularity: Optional[str] = "tensor"+ static_kv_granularity: Optional[str] = "tensor"super_bits: Optional[int] = Nonesuper_group_size: Optional[int] = Noneto_quant_block_names: Optional[list[str]] = None@@ -314,12 +316,18 @@ class BaseOrchestrator(object):if device is not None:logger.warning("`device` is deprecated, please use `device_map` instead")+ from auto_round.experimental.utils import normalize_fp8_granularity+self.static_attention_dtype = kwargs.pop("static_attention_dtype", None)+ self.static_attention_granularity = normalize_fp8_granularity(+ kwargs.pop("static_attention_granularity", "tensor")+ )# Attention static dtypeif self.static_attention_dtype is not None:logger.warning("The static attention dtype is experimental and currently has limited support.")# KV cache, this one does not affect tuning but will collect some infos during tuningself.static_kv_dtype = kwargs.pop("static_kv_dtype", None)+ self.static_kv_granularity = normalize_fp8_granularity(kwargs.pop("static_kv_granularity", "tensor"))if self.static_kv_dtype is not None:logger.warning("The static kv is experimental and currently has limited support.")@@ -432,6 +440,8 @@ class BaseOrchestrator(object):formats=self.formats,static_kv_dtype=self.static_kv_dtype,static_attention_dtype=self.static_attention_dtype,+ static_kv_granularity=self.static_kv_granularity,+ static_attention_granularity=self.static_attention_granularity,)self.shard_writer = None# Resumability state deferred from Orchestrator._quantize_data_driven() until@@ -1832,13 +1842,21 @@ class BaseOrchestrator(object):if self.static_attention_dtype is not None:from auto_round.experimental.attention import attention_quant_ctx- with attention_quant_ctx(self.model_context.model, static_attention_dtype=self.static_attention_dtype):+ with attention_quant_ctx(+ self.model_context.model,+ static_attention_dtype=self.static_attention_dtype,+ static_attention_granularity=self.static_attention_granularity,+ ):self.quantize()self.model_context.quantized = Trueelif self.static_kv_dtype is not None:from auto_round.experimental.kv_cache import kvcache_quant_context- with kvcache_quant_context(self.model_context.model, static_kv_dtype=self.static_kv_dtype):+ with kvcache_quant_context(+ self.model_context.model,+ static_kv_dtype=self.static_kv_dtype,+ static_kv_granularity=self.static_kv_granularity,+ ):self.quantize()self.model_context.quantized = Trueelse:diff --git a/auto_round/context/compress.py b/auto_round/context/compress.pyindex 25704864c..6cb0bfbca 100644--- a/auto_round/context/compress.py+++ b/auto_round/context/compress.py@@ -40,6 +40,8 @@ class CompressContext(BaseContext):output_dir: str = "./compressed_models",static_kv_dtype: Optional[torch.dtype] = None, # TODO later this should be scheme wenhuachstatic_attention_dtype: Optional[torch.dtype] = None,+ static_kv_granularity: str = "tensor",+ static_attention_granularity: str = "tensor",**kwargs,) -> None:super().__init__()@@ -58,6 +60,8 @@ class CompressContext(BaseContext):self.is_immediate_saving = is_immediate_savingself.static_kv_dtype = static_kv_dtypeself.static_attention_dtype = static_attention_dtype+ self.static_kv_granularity = static_kv_granularity+ self.static_attention_granularity = static_attention_granularitydef clear_memory(self, tensor=None):"""Clear GPU/CPU memory only when ``low_gpu_mem_usage`` is enabled."""diff --git a/auto_round/experimental/attention.py b/auto_round/experimental/attention.pyindex ea293873d..49efce384 100644--- a/auto_round/experimental/attention.py+++ b/auto_round/experimental/attention.py@@ -32,8 +32,9 @@ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONSfrom auto_round.experimental.kv_cache import kvcache_quant_contextfrom auto_round.experimental.utils import (clean_model_parameters_and_buffers_,+ fp8_qdq,is_attention_module,- per_tensor_fp8_qdq,+ normalize_fp8_granularity,update_parameter_data,)from auto_round.utils import logger@@ -75,9 +76,10 @@ class QuantizedAttentionImpl(torch.nn.Module):_original_impl = "sdpa"- def __init__(self, config: PretrainedConfig, attn_module: Module):+ def __init__(self, config: PretrainedConfig, attn_module: Module, granularity: str = "tensor"):super().__init__()self.config = config+ self.granularity = normalize_fp8_granularity(granularity)self.attn_module = ref(attn_module) # avoid circular references# register query maxdevice = next(attn_module.parameters()).device@@ -95,14 +97,17 @@ class QuantizedAttentionImpl(torch.nn.Module):*args,**kwargs,):- cur_query_max = query.abs().max()+ if self.granularity == "head":+ cur_query_max = query.abs().amax(dim=(0, 2, 3))+ else:+ cur_query_max = query.abs().max()query_max = torch.max(getattr(module, QUERY_MAX_NAME).data,cur_query_max.detach().to(getattr(module, QUERY_MAX_NAME).data.device),)update_parameter_data(module, query_max, QUERY_MAX_NAME)- _, query_scale = per_tensor_fp8_qdq(query, tensor_max=query_max)- update_parameter_data(module, query_scale.squeeze(0).detach(), QUERY_SCALE_NAME)+ _, query_scale = fp8_qdq(query, tensor_max=query_max, granularity=self.granularity)+ update_parameter_data(module, query_scale.reshape(-1).detach(), QUERY_SCALE_NAME)# original attentionreturn ALL_ATTENTION_FUNCTIONS[self._original_impl](module,@@ -124,7 +129,7 @@ def _ct_hooked_attention(module: Module, *args, **kwargs):return ALL_ATTENTION_FUNCTIONS[_original_impl](module, *args, **kwargs) # pylint: disable=E0601-def init_hooked_attention(module: Module, config):+def init_hooked_attention(module: Module, config, granularity: str = "tensor"):"""Initialize `QuantizedAttentionImpl` and `QuantizedKVCache` instancesattached to attention@@ -133,7 +138,7 @@ def init_hooked_attention(module: Module, config)::param module: attention module to initialize with"""if not hasattr(module, ATTN_IMPL_ATTR_NAME):- module.register_module(ATTN_IMPL_ATTR_NAME, QuantizedAttentionImpl(config, module))+ module.register_module(ATTN_IMPL_ATTR_NAME, QuantizedAttentionImpl(config, module, granularity=granularity))if config._attn_implementation != HOOKED_ATTENTION_NAME:# assumes only one model at a timeglobal _original_impl@@ -145,10 +150,10 @@ def init_hooked_attention(module: Module, config):# initialize_hooked_kv_cache(model, module)-def prep_attention_module_for_calibration(module: torch.nn.Module, config):+def prep_attention_module_for_calibration(module: torch.nn.Module, config, granularity: str = "tensor"):if is_attention_module(module):logger.trace(f"Preparing attention module {module.__class__.__name__} for calibration")- init_hooked_attention(module, config)+ init_hooked_attention(module, config, granularity=granularity)def clean_up_hooked_attention(module, model):@@ -161,12 +166,25 @@ def clean_up_hooked_attention(module, model):@contextlib.contextmanager-def attention_quant_ctx(model: PreTrainedModel, static_attention_dtype=torch.float8_e4m3fn):+def attention_quant_ctx(+ model: PreTrainedModel,+ static_attention_dtype=torch.float8_e4m3fn,+ static_attention_granularity: str = "tensor",+):try:# Setup phase: Initialize hooked attention- prepare_fn = partial(prep_attention_module_for_calibration, config=model.config)+ static_attention_granularity = normalize_fp8_granularity(static_attention_granularity)+ prepare_fn = partial(+ prep_attention_module_for_calibration,+ config=model.config,+ granularity=static_attention_granularity,+ )model.apply(prepare_fn)- with kvcache_quant_context(model, static_kv_dtype=static_attention_dtype):+ with kvcache_quant_context(+ model,+ static_kv_dtype=static_attention_dtype,+ static_kv_granularity=static_attention_granularity,+ ):yield modelfinally:clean_fn = partial(clean_up_hooked_attention, model=model)diff --git a/auto_round/experimental/kv_cache.py b/auto_round/experimental/kv_cache.pyindex 1774315d7..ba35c408e 100644--- a/auto_round/experimental/kv_cache.py+++ b/auto_round/experimental/kv_cache.py@@ -25,9 +25,10 @@ import torchfrom transformers.cache_utils import DynamicCachefrom auto_round.experimental.utils import (+ fp8_qdq,is_attention_module,+ normalize_fp8_granularity,normalize_static_kv_dtype,- per_tensor_fp8_qdq,update_parameter_data,)from auto_round.utils import logger@@ -106,9 +107,10 @@ class QuantizedKVParameterCache(DynamicCache):cls._instance = super(QuantizedKVParameterCache, cls).__new__(cls)return cls._instance- def __init__(self, dtype: torch.dtype = torch.float8_e4m3fn):+ def __init__(self, dtype: torch.dtype = torch.float8_e4m3fn, granularity: str = "tensor"):assert dtype == torch.float8_e4m3fn, "Only fp8_e4m3fn is supported for now."+ self.granularity = normalize_fp8_granularity(granularity)if not self._initialized:super().__init__()@@ -172,13 +174,13 @@ class QuantizedKVParameterCache(DynamicCache):assert kv_type == KVCacheScaleType.VALUEscales = self.v_scales- qdq_tensor, scale = per_tensor_fp8_qdq(tensor)+ qdq_tensor, scale = fp8_qdq(tensor, granularity=self.granularity)# Detach scale to prevent holding computation graph references- _pad_and_append_at_idx_(scales, layer_idx, scale.squeeze(0).detach())+ _pad_and_append_at_idx_(scales, layer_idx, scale.reshape(-1).detach())return qdq_tensor-def initialize_quantized_kv_cache(module: torch.nn.Module, dtype=torch.float8_e4m3fn):+def initialize_quantized_kv_cache(module: torch.nn.Module, dtype=torch.float8_e4m3fn, granularity: str = "tensor"):"""Initialize a quantized kv_cache on a module (analogous to initializing an observer)"""@@ -189,7 +191,7 @@ def initialize_quantized_kv_cache(module: torch.nn.Module, dtype=torch.float8_e4if isinstance(existing_kv_cache, QuantizedKVParameterCache):return- quantized_kv_cache = QuantizedKVParameterCache(dtype=dtype)+ quantized_kv_cache = QuantizedKVParameterCache(dtype=dtype, granularity=granularity)setattr(module, "kv_cache", quantized_kv_cache)logger.debug(f"Initialized quantized kv_cache for {module.__class__.__name__} {getattr(module, 'layer_idx', None)}")init_scale = torch.tensor([0.0], device=next(module.parameters()).device)@@ -234,15 +236,20 @@ def prep_attention_module_for_calibration(module: torch.nn.Module):@contextlib.contextmanager-def kvcache_quant_context(model: torch.nn.Module, static_kv_dtype=torch.float8_e4m3fn):+def kvcache_quant_context(+ model: torch.nn.Module, static_kv_dtype=torch.float8_e4m3fn, static_kv_granularity: str = "tensor"+):"""Context manager for FP8 KV cache quantization operations."""try:# Setup phase: Initialize KV cache for quantizationstatic_kv_dtype = normalize_static_kv_dtype(static_kv_dtype)+ static_kv_granularity = normalize_fp8_granularity(static_kv_granularity)if static_kv_dtype != torch.float8_e4m3fn:logger.warning(f"Ignoring static kv dtype {static_kv_dtype}, only fp8_e4m3fn is supported.")else:- initialize_fn = partial(initialize_quantized_kv_cache, dtype=static_kv_dtype)+ initialize_fn = partial(+ initialize_quantized_kv_cache, dtype=static_kv_dtype, granularity=static_kv_granularity+ )model.apply(initialize_fn)model.apply(prep_attention_module_for_calibration)diff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.pyindex c9a5dd2aa..2b106f1f1 100644--- a/auto_round/experimental/utils.py+++ b/auto_round/experimental/utils.py@@ -23,6 +23,22 @@ from auto_round.utils import loggerSUPPORTED_QUANTIZATION_SCHEMES = ["MXFP8", "MXFP4", "NVFP4"]+FP8_GRANULARITY_TENSOR = "tensor"+FP8_GRANULARITY_HEAD = "head"+++def normalize_fp8_granularity(granularity: str | None) -> str:+ if granularity is None:+ return FP8_GRANULARITY_TENSOR+ granularity = granularity.lower()+ if granularity in (FP8_GRANULARITY_TENSOR, FP8_GRANULARITY_HEAD):+ return granularity+ raise ValueError(+ f"Invalid FP8 calibration granularity: {granularity}. "+ f"Supported granularities are: {FP8_GRANULARITY_TENSOR}, {FP8_GRANULARITY_HEAD}."+ )++def per_tensor_fp8_qdq(tensor: torch.Tensor, tensor_max: None | torch.Tensor = None) -> tuple[torch.Tensor, torch.Tensor]:@@ -32,6 +48,41 @@ def per_tensor_fp8_qdq(return qdq_tensor, scale+def per_head_fp8_qdq(tensor: torch.Tensor, tensor_max: None | torch.Tensor = None) -> tuple[torch.Tensor, torch.Tensor]:+ """Quantize/dequantize an attention tensor with one FP8 scale per attention head."""+ if tensor.dim() < 4:+ raise ValueError(f"Per-head FP8 calibration expects [batch, heads, seq, head_dim], got {tuple(tensor.shape)}")+ if tensor_max is not None and tensor_max.shape != (tensor.shape[1],):+ raise ValueError(+ f"Per-head FP8 calibration expects tensor_max shape ({tensor.shape[1]},), " f"got {tuple(tensor_max.shape)}"+ )++ info = torch.finfo(torch.float8_e4m3fn)+ orig_dtype = tensor.dtype+ if tensor.dtype == torch.float16:+ tensor = tensor.to(torch.bfloat16)++ if tensor_max is None:+ max_tensor = tensor.abs().amax(dim=(0, 2, 3)).to(torch.float32)+ else:+ max_tensor = tensor_max.to(tensor.device, dtype=torch.float32)++ scale = torch.clip(max_tensor / info.max, min=float(1.0 / (info.max * 512.0)))+ qdq_scale = scale.view(1, -1, 1, 1)+ fp8_res = torch.clip(tensor / qdq_scale, info.min, info.max)+ fp8_res = fp8_res.to(torch.float8_e4m3fn)+ return (fp8_res.to(tensor.dtype) * qdq_scale).to(orig_dtype), scale+++def fp8_qdq(+ tensor: torch.Tensor, tensor_max: None | torch.Tensor = None, granularity: str | None = None+) -> tuple[torch.Tensor, torch.Tensor]:+ granularity = normalize_fp8_granularity(granularity)+ if granularity == FP8_GRANULARITY_HEAD:+ return per_head_fp8_qdq(tensor, tensor_max=tensor_max)+ return per_tensor_fp8_qdq(tensor, tensor_max=tensor_max)++@torch.compiler.disabledef update_parameter_data(module: torch.nn.Module, new_val: torch.Tensor, name: str):"""@@ -41,7 +92,17 @@ def update_parameter_data(module: torch.nn.Module, new_val: torch.Tensor, name:if hasattr(module, name):param = getattr(module, name)if isinstance(param, torch.nn.Parameter):- param.data.copy_(new_val)+ if param.shape == new_val.shape:+ param.data.copy_(new_val)+ else:+ logger.warning(+ "Replacing parameter %s in module %s with a new shape: %s -> %s",+ name,+ module.__class__.__name__,+ tuple(param.shape),+ tuple(new_val.shape),+ )+ setattr(module, name, torch.nn.Parameter(new_val))else:module.register_parameter(name, torch.nn.Parameter(new_val))else:diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex d43a0962e..c061095da 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -161,6 +161,8 @@ def _build_mixed_fp_quantization_config(model,static_kv_dtype=None,static_attention_dtype=None,+ static_kv_granularity="tensor",+ static_attention_granularity="tensor",):"""Build a quantization config dict for mixed-precision scenarios.@@ -204,13 +206,16 @@ def _build_mixed_fp_quantization_config(use_fp8_attention = _use_fp8_attention(static_attention_dtype)if use_fp8_attention:- attention_config = _get_attention_config(model)+ attention_config = _get_attention_config(model, static_attention_granularity)else:attention_config = None+ kv_granularity = static_attention_granularity if use_fp8_attention else static_kv_granularityquantization_config = initialize_quantization(scheme=None,config_groups=config_groups,- kv_cache_scheme=_construct_kv_scheme() if (_use_fp8_kv(static_kv_dtype) or use_fp8_attention) else None,+ kv_cache_scheme=(+ _construct_kv_scheme(kv_granularity) if (_use_fp8_kv(static_kv_dtype) or use_fp8_attention) else None+ ),ignore=ignore,)quantization_config = quantization_config.to_dict()@@ -322,8 +327,11 @@ def save_quantized_as_fp(is_mixed = len(scheme_groups) > 1use_fp8_attention = _use_fp8_attention(serialization_dict.get("static_attention_dtype", None))+ static_attention_granularity = serialization_dict.get("static_attention_granularity", "tensor")+ static_kv_granularity = serialization_dict.get("static_kv_granularity", "tensor")+ kv_granularity = static_attention_granularity if use_fp8_attention else static_kv_granularitykv_cache_scheme = (- _construct_kv_scheme()+ _construct_kv_scheme(kv_granularity)if (_use_fp8_kv(serialization_dict.get("static_kv_dtype", None)) or use_fp8_attention)else None)@@ -338,6 +346,8 @@ def save_quantized_as_fp(model,static_kv_dtype=serialization_dict.get("static_kv_dtype", None),static_attention_dtype=serialization_dict.get("static_attention_dtype", None),+ static_kv_granularity=static_kv_granularity,+ static_attention_granularity=static_attention_granularity,)else:scheme = _get_scheme(bits, data_type)@@ -351,7 +361,7 @@ def save_quantized_as_fp(ignore=ignore,)if use_fp8_attention:- attention_config = _get_attention_config(model)+ attention_config = _get_attention_config(model, static_attention_granularity)else:attention_config = Nonesetattr(quantization_config, "format", format)diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex 82cf6d2a2..a2cc52b07 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -62,25 +62,32 @@ def pack_layer(layer_name: str, model: torch.nn.Module, data_type: str, device:fp8_pack_layer(layer_name, model, data_type, device, unsqueeze=True)-def _construct_fp8_args():+def _get_fp8_strategy(granularity: str = "tensor"):+ from compressed_tensors.quantization import QuantizationStrategy # pylint: disable=E0401++ if granularity == "head":+ return QuantizationStrategy.ATTN_HEAD+ return QuantizationStrategy.TENSOR+++def _construct_fp8_args(granularity: str = "tensor"):from compressed_tensors.quantization import ( # pylint: disable=E0401QuantizationArgs,- QuantizationStrategy,QuantizationType,)return QuantizationArgs(num_bits=8,type=QuantizationType.FLOAT,- strategy=QuantizationStrategy.TENSOR,+ strategy=_get_fp8_strategy(granularity),symmetric=True,dynamic=False,)-def _construct_kv_scheme():+def _construct_kv_scheme(granularity: str = "tensor"):"""Construct the default KV cache quantization scheme for FP8_STATIC export."""- default_kv_scheme = _construct_fp8_args()+ default_kv_scheme = _construct_fp8_args(granularity)logger.warning_once("Using default KV cache scheme: %s. "@@ -119,7 +126,7 @@ def _get_attention_targets(model: torch.nn.Module) -> list[str]:return attention_targets-def _get_attention_config(model: torch.nn.Module) -> dict | None:+def _get_attention_config(model: torch.nn.Module, granularity: str = "tensor") -> dict | None:"""Return attention FP8 config as a standalone dict for the top-level``attention_input_activations`` field.@@ -134,7 +141,7 @@ def _get_attention_config(model: torch.nn.Module) -> dict | None:return {"targets": attention_targets,- "input_activations": _construct_fp8_args().model_dump(),+ "input_activations": _construct_fp8_args(granularity).model_dump(),}@@ -237,18 +244,15 @@ def save_quantized_as_static_fp(config_groups = {}scheme = QuantizationScheme(targets=targets, **scheme_args)config_groups["group_0"] = scheme+ static_attention_granularity = serialization_dict.get("static_attention_granularity", "tensor")+ static_kv_granularity = serialization_dict.get("static_kv_granularity", "tensor")use_fp8_attention = _use_fp8_attention(serialization_dict.get("static_attention_dtype", None))- if use_fp8_attention:- attention_config = _get_attention_config(model)- else:- attention_config = None+ use_fp8_kv = _use_fp8_kv(serialization_dict.get("static_kv_dtype", None))+ attention_config = _get_attention_config(model, static_attention_granularity) if use_fp8_attention else None+ kv_granularity = static_attention_granularity if use_fp8_attention else static_kv_granularityquantization_config = QuantizationConfig(config_groups=config_groups,- kv_cache_scheme=(- _construct_kv_scheme()- if (_use_fp8_kv(serialization_dict.get("static_kv_dtype", None)) or use_fp8_attention)- else None- ),+ kv_cache_scheme=_construct_kv_scheme(kv_granularity) if (use_fp8_kv or use_fp8_attention) else None,quantization_status=QuantizationStatus.COMPRESSED,ignore=ignore,)diff --git a/test/unit/test_cpu/export/test_export.py b/test/unit/test_cpu/export/test_export.pyindex 805e68675..48c8c7a7b 100644--- a/test/unit/test_cpu/export/test_export.py+++ b/test/unit/test_cpu/export/test_export.py@@ -224,11 +224,20 @@ class TestAutoRound:)quantized_model_path = self.save_dir_, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")- assert "model.decoder.layers.0.self_attn.k_proj.input_scale" in f.keys()- assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()- assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.input_scale").shape == torch.Size([1])- assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.weight").dtype == torch.float8_e4m3fn+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:+ assert "model.decoder.layers.0.self_attn.k_proj.input_scale" in f.keys()+ assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.input_scale").shape == torch.Size([1])+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.weight").dtype == torch.float8_e4m3fn+ if static_kv_dtype == "fp8":+ assert "model.decoder.layers.0.self_attn.k_scale" in f.keys()+ assert "model.decoder.layers.0.self_attn.v_scale" in f.keys()+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_scale").shape == torch.Size([1])+ assert f.get_tensor("model.decoder.layers.0.self_attn.k_scale").shape == torch.Size([1])+ assert (+ f.get_tensor("model.decoder.layers.0.self_attn.k_scale").dtype == torch.float32+ or f.get_tensor("model.decoder.layers.0.self_attn.k_scale").dtype == torch.bfloat16+ )if static_kv_dtype is None:with torch.no_grad():import transformers@@ -243,7 +252,10 @@ class TestAutoRound:assert (model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__== "WeightFP8ActFP8StaticQuantLinear"- ), f"Expected WeightFP8ActFP8StaticQuantLinear, got {model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__}"+ ), (+ "Expected WeightFP8ActFP8StaticQuantLinear, "+ f"got {model.model.decoder.layers[0].self_attn.k_proj.__class__.__name__}"+ )tokenizer = transformers.AutoTokenizer.from_pretrained(quantized_model_path)prompt = "AI is "encode = tokenizer.encode(prompt, return_tensors="pt")@@ -257,16 +269,6 @@ class TestAutoRound:print(f"Output: {output}")assert output is not None, "Output should not be None"- if static_kv_dtype == "fp8":- assert "model.decoder.layers.0.self_attn.k_scale" in f.keys()- assert "model.decoder.layers.0.self_attn.v_scale" in f.keys()- assert f.get_tensor("model.decoder.layers.0.self_attn.v_scale").shape == torch.Size([1])- assert f.get_tensor("model.decoder.layers.0.self_attn.k_scale").shape == torch.Size([1])- assert (- f.get_tensor("model.decoder.layers.0.self_attn.k_scale").dtype == torch.float32- or f.get_tensor("model.decoder.layers.0.self_attn.k_scale").dtype == torch.bfloat16- )-model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)autoround = AutoRound(model,@@ -285,11 +287,49 @@ class TestAutoRound:quantized_model_path = self.save_dir_, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:+ assert "model.decoder.layers.0.self_attn.k_proj.input_scale" in f.keys()+ assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.input_scale").shape == torch.Size([1])+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.weight").dtype == torch.float8_e4m3fn++ def test_static_afp8_per_head_export(self):+ import os++ from safetensors import safe_open++ model_name = self.model_name+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=8,+ group_size=-1,+ iters=0,+ scheme="fp8_static",+ nsamples=2,+ seqlen=2,+ static_kv_dtype="fp8",+ static_kv_granularity="head",+ )+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")assert "model.decoder.layers.0.self_attn.k_proj.input_scale" in f.keys()assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.input_scale").shape == torch.Size([1])assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.weight").dtype == torch.float8_e4m3fn+ assert f.get_tensor("model.decoder.layers.0.self_attn.k_scale").shape == torch.Size(+ [model.config.num_attention_heads]+ )+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_scale").shape == torch.Size(+ [model.config.num_attention_heads]+ )++ with open(os.path.join(quantized_model_path, "config.json")) as config_file:+ config = json.load(config_file)+ quantization_config = config["quantization_config"]+ assert quantization_config["static_kv_dtype"] == "fp8"+ assert quantization_config["static_kv_granularity"] == "head"def test_static_fp8_attn(self):import os@@ -309,18 +349,51 @@ class TestAutoRound:)quantized_model_path = self.save_dir_, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:+ assert "model.decoder.layers.0.self_attn.k_proj.input_scale" in f.keys()+ assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.input_scale").shape == torch.Size([1])+ assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.weight").dtype == torch.float8_e4m3fn+ check_attrs = ["k_scale", "v_scale", "q_scale"]+ for attr in check_attrs:+ weight_name = f"model.decoder.layers.0.self_attn.{attr}"+ assert weight_name in f.keys()+ assert f.get_tensor(weight_name).shape == torch.Size([1])+ assert (+ f.get_tensor(weight_name).dtype == torch.float32+ or f.get_tensor(weight_name).dtype == torch.bfloat16+ )+ assert not any(key.endswith(".q_max") for key in f.keys())++ def test_static_fp8_per_head_attn(self):+ import os++ from safetensors import safe_open++ model_name = self.model_name+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ autoround = AutoRound(+ model,+ self.tokenizer,+ iters=0,+ nsamples=2,+ seqlen=2,+ scheme="FP8_STATIC",+ static_attention_dtype="fp8",+ static_attention_granularity="head",+ )+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")- assert "model.decoder.layers.0.self_attn.k_proj.input_scale" in f.keys()- assert "model.decoder.layers.0.self_attn.k_proj.weight_scale" in f.keys()- assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.input_scale").shape == torch.Size([1])- assert f.get_tensor("model.decoder.layers.0.self_attn.v_proj.weight").dtype == torch.float8_e4m3fn- check_attrs = ["k_scale", "v_scale", "q_scale"]- for attr in check_attrs:+ for attr in ("k_scale", "v_scale", "q_scale"):weight_name = f"model.decoder.layers.0.self_attn.{attr}"assert weight_name in f.keys()- assert f.get_tensor(weight_name).shape == torch.Size([1])- assert f.get_tensor(weight_name).dtype == torch.float32 or f.get_tensor(weight_name).dtype == torch.bfloat16- assert "model.decoder.layers.8.self_attn.q_max" not in f.keys()+ assert f.get_tensor(weight_name).shape == torch.Size([model.config.num_attention_heads])++ with open(os.path.join(quantized_model_path, "config.json")) as config_file:+ config = json.load(config_file)+ quantization_config = config["quantization_config"]+ assert quantization_config["static_attention_dtype"] == "fp8"+ assert quantization_config["static_attention_granularity"] == "head"@pytest.mark.timeout(120)def test_awq_lmhead_export(self, dataloader):diff --git a/test/unit/test_cpu/export/test_llmc_format.py b/test/unit/test_cpu/export/test_llmc_format.pyindex cf83d8b9f..f1725b32b 100644--- a/test/unit/test_cpu/export/test_llmc_format.py+++ b/test/unit/test_cpu/export/test_llmc_format.py@@ -152,6 +152,35 @@ class TestLLMC:assert kv_cache_scheme["dynamic"] is Falseassert kv_cache_scheme["symmetric"] is True+ def test_mxfp8_llmcompressor_per_head_kv_config(self, tiny_opt_model_path, tmp_path):+ from safetensors import safe_open++ ar = AutoRound(+ model=tiny_opt_model_path,+ iters=0,+ disable_opt_rtn=True,+ scheme="mxfp8",+ static_kv_dtype="fp8",+ static_kv_granularity="head",+ )+ compressed_model, quantized_model_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")++ with open(os.path.join(quantized_model_path, "config.json")) as f:+ config = json.load(f)++ kv_cache_scheme = config["quantization_config"]["kv_cache_scheme"]+ assert kv_cache_scheme is not None+ assert kv_cache_scheme["strategy"] == "attn_head"++ num_kv_heads = compressed_model.config.num_attention_heads+ if hasattr(compressed_model.config, "num_key_value_heads"):+ num_kv_heads = compressed_model.config.num_key_value_heads+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:+ k_scale = f.get_tensor("model.decoder.layers.0.self_attn.k_scale")+ v_scale = f.get_tensor("model.decoder.layers.0.self_attn.v_scale")+ assert k_scale.shape == torch.Size([num_kv_heads])+ assert v_scale.shape == torch.Size([num_kv_heads])+def test_mxfp8_llmcompressor_attention_config(self, tiny_opt_model_path, tmp_path):from safetensors import safe_open@@ -194,6 +223,34 @@ class TestLLMC:assert any(key.endswith(".self_attn.q_scale") for key in keys), "q_scale not found in checkpoint"assert not any(key.endswith(".q_max") for key in keys), "q_max should not be exported"+ def test_mxfp8_llmcompressor_per_head_attention_config(self, tiny_opt_model_path, tmp_path):+ from safetensors import safe_open++ ar = AutoRound(+ model=tiny_opt_model_path,+ iters=0,+ disable_opt_rtn=True,+ scheme="mxfp8",+ static_attention_dtype="fp8",+ static_attention_granularity="head",+ )+ compressed_model, quantized_model_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")++ with open(os.path.join(quantized_model_path, "config.json")) as f:+ saved_config = json.load(f)++ attention_config = saved_config["quantization_config"]["attention_input_activations"]+ assert attention_config is not None+ assert attention_config["targets"] == [compressed_model.model.decoder.layers[0].self_attn.__class__.__name__]+ assert attention_config["input_activations"]["strategy"] == "attn_head"+ assert saved_config["quantization_config"]["kv_cache_scheme"]["strategy"] == "attn_head"++ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:+ q_scale = f.get_tensor("model.decoder.layers.0.self_attn.q_scale")+ k_scale = f.get_tensor("model.decoder.layers.0.self_attn.k_scale")+ assert q_scale.shape == torch.Size([compressed_model.config.num_attention_heads])+ assert k_scale.ndim == 1+@pytest.mark.timeout(60)def test_mixed_precision_llmcompressor_format(self, tiny_opt_model_path, tmp_path):scheme = AutoScheme(diff --git a/test/unit/test_cpu/test_experimental_fp8_granularity.py b/test/unit/test_cpu/test_experimental_fp8_granularity.pynew file mode 100644index 000000000..61bcfcf51--- /dev/null+++ b/test/unit/test_cpu/test_experimental_fp8_granularity.py@@ -0,0 +1,29 @@+import pytest+import torch++from auto_round.experimental.utils import fp8_qdq+++def test_per_head_fp8_qdq_preserves_mha_head_axis():+ tensor = torch.randn(2, 4, 3, 8)++ qdq_tensor, scale = fp8_qdq(tensor, granularity="head")++ assert qdq_tensor.shape == tensor.shape+ assert scale.shape == torch.Size([4])+++def test_per_head_fp8_qdq_preserves_gqa_kv_head_axis():+ tensor = torch.randn(2, 2, 3, 8)++ qdq_tensor, scale = fp8_qdq(tensor, granularity="head")++ assert qdq_tensor.shape == tensor.shape+ assert scale.shape == torch.Size([2])+++def test_per_head_fp8_qdq_rejects_mismatched_tensor_max_shape():+ tensor = torch.randn(2, 4, 3, 8)++ with pytest.raises(ValueError, match=r"tensor_max shape \(4,\)"):+ fp8_qdq(tensor, tensor_max=torch.ones(2), granularity="head")