PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
feat: add NVFP4 static KV cache quantization support (#2366)
Yi Liu04fca6225906
diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.pyindex 77793f1b4..bad251a7a 100644--- a/auto_round/cli/parser.py+++ b/auto_round/cli/parser.py@@ -244,7 +244,7 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu"--static_kv_dtype",default=None,type=str,- choices=["fp8", "float8_e4m3fn"],+ choices=["fp8", "float8_e4m3fn", "nvfp4"],help="Static KV-cache quantization data type.",)rt.add_argument(diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex fc722ef53..7beb61ccf 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -554,6 +554,12 @@ class BaseOrchestrator(object):if isinstance(self.scheme, AutoScheme):return True+ # Static KV-cache / attention quantization observes KV magnitudes+ # during the calibration forwards, so it always needs data — even for+ # weight-only schemes (e.g. NVFP4 with iters=0).+ if self.static_kv_dtype is not None or self.static_attention_dtype is not None:+ return True+# Check if activation calibration is neededfrom auto_round.compressors.utils import check_need_act_calibrationdiff --git a/auto_round/experimental/kv_cache.py b/auto_round/experimental/kv_cache.pyindex 5bee3e8c0..948a48f91 100644--- a/auto_round/experimental/kv_cache.py+++ b/auto_round/experimental/kv_cache.py@@ -25,6 +25,9 @@ import torchfrom transformers.cache_utils import DynamicCachefrom auto_round.experimental.utils import (+ FP8_GRANULARITY_TENSOR,+ NVFP4_KV_BLOCK_SIZE,+ NVFP4_KV_DTYPE,fp8_qdq,is_attention_module,normalize_fp8_granularity,@@ -107,16 +110,25 @@ class QuantizedKVParameterCache(DynamicCache):cls._instance = super(QuantizedKVParameterCache, cls).__new__(cls)return cls._instance- def __init__(self, dtype: torch.dtype = torch.float8_e4m3fn, granularity: str = "tensor"):-- assert dtype == torch.float8_e4m3fn, "Only fp8_e4m3fn is supported for now."- self.granularity = normalize_fp8_granularity(granularity)+ def __init__(self, dtype: torch.dtype | str = torch.float8_e4m3fn, granularity: str = "tensor"):+ dtype = normalize_static_kv_dtype(dtype)+ self.is_nvfp4 = dtype == NVFP4_KV_DTYPE+ if self.is_nvfp4:+ # NVFP4 block scales are computed per 16-element group at runtime;+ # only a static per-tensor global scale is calibrated, so "head"+ # granularity does not apply.+ self.granularity = FP8_GRANULARITY_TENSOR+ else:+ assert dtype == torch.float8_e4m3fn, "Only fp8_e4m3fn is supported for now."+ self.granularity = normalize_fp8_granularity(granularity)if not self._initialized:super().__init__()# each index corresponds to layer_idx of the attention layerself.k_scales: List[torch.Tensor] = []self.v_scales: List[torch.Tensor] = []+ self.k_amax: List[float] = []+ self.v_amax: List[float] = []self._initialized = Truedef update(@@ -129,6 +141,13 @@ class QuantizedKVParameterCache(DynamicCache):"""Get the k_scale and v_scale and output the quant-dequant key_states and value_states"""+ if self.is_nvfp4:+ # Observe-only: track the per-side running amax and let K/V+ # pass through unmodified (see _nvfp4_track_amax).+ self._nvfp4_track_amax(key_states, KVCacheScaleType.KEY, layer_idx)+ self._nvfp4_track_amax(value_states, KVCacheScaleType.VALUE, layer_idx)+ return key_states, value_states+qdq_key_states = self._quant_dequant(key_states.contiguous(), KVCacheScaleType.KEY, layer_idx)qdq_value_states = self._quant_dequant(value_states.contiguous(), KVCacheScaleType.VALUE, layer_idx)@@ -181,6 +200,32 @@ class QuantizedKVParameterCache(DynamicCache):_pad_and_append_at_idx_(scales, layer_idx, scale.reshape(-1).detach())return qdq_tensor+ def _nvfp4_track_amax(self, tensor: torch.Tensor, kv_type: KVCacheScaleType, layer_idx: int):+ """Track the running NVFP4 K/V amax during calibration (observe-only).++ Mirrors the observer-based KV calibration in llm-compressor: the+ per-side running amax is accumulated while the K/V tensors pass+ through unmodified, so calibration never fake-quantizes with an+ incomplete scale. The static global scale is derived from the+ final (full-calibration-set) amax by the output hook, which+ writes the vLLM dequant-multiplier convention (``amax / 2688``) --+ see ``_nvfp4_global_scale``.+ """+ if tensor.shape[-1] % NVFP4_KV_BLOCK_SIZE != 0:+ raise ValueError(+ f"NVFP4 KV cache quantization requires the last dim (head_dim) "+ f"to be divisible by {NVFP4_KV_BLOCK_SIZE} (got {tensor.shape[-1]} "+ f"at layer_idx={layer_idx}); use static_kv_dtype='fp8' for "+ f"this model."+ )++ amax = tensor.abs().max().item()+ amax_list = self.k_amax if kv_type == KVCacheScaleType.KEY else self.v_amax+ if layer_idx >= len(amax_list):+ _pad_and_append_at_idx_(amax_list, layer_idx, amax)+ else:+ amax_list[layer_idx] = max(amax_list[layer_idx], amax)+def initialize_quantized_kv_cache(module: torch.nn.Module, dtype=torch.float8_e4m3fn, granularity: str = "tensor"):"""@@ -193,9 +238,28 @@ def initialize_quantized_kv_cache(module: torch.nn.Module, dtype=torch.float8_e4if isinstance(existing_kv_cache, QuantizedKVParameterCache):return+ dtype = normalize_static_kv_dtype(dtype)+ if dtype == NVFP4_KV_DTYPE:+ # NVFP4 KV needs head_dim divisible by the fp4 block size; fail+ # early per layer instead of silently degrading all layers.+ head_dim = getattr(module, "head_dim", None)+ if head_dim is not None and head_dim % NVFP4_KV_BLOCK_SIZE != 0:+ raise ValueError(+ f"NVFP4 KV cache quantization requires head_dim divisible by "+ f"{NVFP4_KV_BLOCK_SIZE} (got {head_dim} in "+ f"{module.__class__.__name__}, "+ f"layer_idx={getattr(module, 'layer_idx', '?')}); "+ f"use static_kv_dtype='fp8' for this model."+ )+quantized_kv_cache = QuantizedKVParameterCache(dtype=dtype, granularity=granularity)setattr(module, "kv_cache", quantized_kv_cache)logger.debug(f"Initialized quantized kv_cache for {module.__class__.__name__} {getattr(module, 'layer_idx', None)}")+ if quantized_kv_cache.is_nvfp4:+ # Global scales are only registered once calibration has observed KV+ # magnitudes; creating zero/placeholder scales up front would export a+ # broken scheme when no calibration forward happens.+ returninit_scale = torch.tensor([0.0], device=next(module.parameters()).device)update_parameter_data(module, init_scale.clone(), KVCacheScaleType.KEY.value)update_parameter_data(module, init_scale.clone(), KVCacheScaleType.VALUE.value)@@ -220,11 +284,52 @@ def calibrate_kv_cache_input_hook(return args, kwargs+def _nvfp4_global_scale(amax: float, device: torch.device) -> torch.Tensor:+ """Static NVFP4 KV scale for the checkpoint, in the vLLM serving convention.++ vLLM's NVFP4 KV cache store kernel (``reshape_and_cache_nvfp4``) treats the+ checkpoint value as the *dequantization multiplier*: it computes+ ``global_scale = 1 / k_scale`` and the per-16-element fp8 block scale as+ ``sf = global_scale * block_max / 6``, then reads back+ ``x = fp4 * sf * k_scale``. Storing the weight-style ``2688 / amax``+ instead would make the runtime build fp8 block scales from+ ``block_max * amax / (6 * 2688)``, which underflows e4m3 (min subnormal+ ~2**-9) for typical activation magnitudes and zeroes the KV cache.++ The reciprocal ``amax / 2688 = 1 / calculate_gparam(amax)`` makes the+ runtime's block scale ``(2688 / amax) * block_max / 6`` -- exactly the+ value auto-round's QDQ simulation (``nv_fp4_with_static_gs``) uses during+ calibration, so served outputs match calibration up to fp8 rounding.++ Note: the pure-Python compressed-tensors runtime (``forward_quantize``)+ interprets the stored value the other way around (``sf = gs * block_max /+ 6``). The two runtimes use opposite conventions; vLLM is the target+ serving engine, hence this convention.+ """+ from auto_round.data_type.nvfp import calculate_gparam++ global_scale = calculate_gparam(amax, device=device)+ return (1.0 / global_scale).reshape(1).detach()++def calibrate_kv_cache_output_hook(module: torch.nn.Module, _args: Any, _output: torch.Tensor):"""Hook to update k_scale and v_scale parameters when running kv_cache quantization."""kv_cache = getattr(module, "kv_cache")+ if kv_cache.is_nvfp4:+ layer_idx = module.layer_idx+ k_amax = kv_cache.k_amax[layer_idx] if layer_idx < len(kv_cache.k_amax) else 0.0+ v_amax = kv_cache.v_amax[layer_idx] if layer_idx < len(kv_cache.v_amax) else 0.0+ if k_amax > 0:+ update_parameter_data(+ module, _nvfp4_global_scale(k_amax, next(module.parameters()).device), "k_global_scale"+ )+ if v_amax > 0:+ update_parameter_data(+ module, _nvfp4_global_scale(v_amax, next(module.parameters()).device), "v_global_scale"+ )+ returnk_scale = kv_cache.k_scales[module.layer_idx]v_scale = kv_cache.v_scales[module.layer_idx]update_parameter_data(module, k_scale, KVCacheScaleType.KEY.value)@@ -241,17 +346,27 @@ def prep_attention_module_for_calibration(module: torch.nn.Module):def kvcache_quant_context(model: torch.nn.Module, static_kv_dtype=torch.float8_e4m3fn, static_kv_granularity: str = "tensor"):- """Context manager for FP8 KV cache quantization operations."""+ """Context manager for static KV cache quantization (FP8 or NVFP4) operations."""try:# Setup phase: Initialize KV cache for quantizationstatic_kv_dtype = normalize_static_kv_dtype(static_kv_dtype)static_kv_granularity = normalize_fp8_granularity(static_kv_granularity)- if static_kv_dtype != torch.float8_e4m3fn:- logger.warning(f"Ignoring static kv dtype {static_kv_dtype}, only fp8_e4m3fn is supported.")+ if static_kv_dtype == NVFP4_KV_DTYPE:+ if static_kv_granularity != FP8_GRANULARITY_TENSOR:+ logger.warning(+ "NVFP4 KV cache only supports 'tensor' granularity; ignoring granularity %r.",+ static_kv_granularity,+ )+ initialize_fn = partial(+ initialize_quantized_kv_cache, dtype=NVFP4_KV_DTYPE, granularity=FP8_GRANULARITY_TENSOR+ )+ elif static_kv_dtype != torch.float8_e4m3fn:+ logger.warning(f"Ignoring static kv dtype {static_kv_dtype}, only fp8_e4m3fn and nvfp4 are supported.")else:initialize_fn = partial(initialize_quantized_kv_cache, dtype=static_kv_dtype, granularity=static_kv_granularity)+ if static_kv_dtype in (torch.float8_e4m3fn, NVFP4_KV_DTYPE):model.apply(initialize_fn)model.apply(prep_attention_module_for_calibration)diff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.pyindex 2b106f1f1..2391d1798 100644--- a/auto_round/experimental/utils.py+++ b/auto_round/experimental/utils.py@@ -112,8 +112,19 @@ def update_parameter_data(module: torch.nn.Module, new_val: torch.Tensor, name:module.register_parameter(name, torch.nn.Parameter(new_val))-def normalize_static_kv_dtype(static_kv_dtype: str | torch.dtype) -> torch.dtype:- valid_dtype_name_lst = ["float16", "bfloat16", "fp8", "float32", "float"]+NVFP4_KV_DTYPE = "nvfp4"+NVFP4_KV_BLOCK_SIZE = 16+++def normalize_static_kv_dtype(static_kv_dtype: str | torch.dtype) -> torch.dtype | str:+ """Normalize the static KV-cache quantization dtype.++ Returns a ``torch.dtype`` for float dtypes, or the ``"nvfp4"`` sentinel+ (see :data:`NVFP4_KV_DTYPE`) for NVFP4 KV cache quantization.+ """+ if isinstance(static_kv_dtype, torch.dtype):+ return static_kv_dtype+ valid_dtype_name_lst = ["float16", "bfloat16", "fp8", "float32", "float", NVFP4_KV_DTYPE, "fp4", "float4_e2m1"]valid_torch_dtype = {"float16": torch.float16,"bfloat16": torch.bfloat16,@@ -122,6 +133,8 @@ def normalize_static_kv_dtype(static_kv_dtype: str | torch.dtype) -> torch.dtype"float32": torch.float32,"float": torch.float32, # Alias for float32}+ if static_kv_dtype in (NVFP4_KV_DTYPE, "fp4", "float4_e2m1"):+ return NVFP4_KV_DTYPEif static_kv_dtype in valid_dtype_name_lst:new_dtype = valid_torch_dtype[static_kv_dtype]elif static_kv_dtype in valid_torch_dtype.values():diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex f207a035d..9746b7e46 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -54,9 +54,11 @@ from .config import check_compressed_tensors_supportedfrom .export_to_static_fp import (_configure_gaudi2_fp8_dtype,_construct_kv_scheme,+ _construct_nvfp4_kv_scheme,_get_attention_config,_use_fp8_attention,_use_fp8_kv,+ _use_nvfp4_kv,)__all__ = [@@ -156,6 +158,27 @@ def _get_group_format(bits, data_type):return "float-quantized"+def _attention_modules_have_nvfp4_kv_scales(model: torch.nn.Module) -> bool:+ """True when every attention module collected NVFP4 KV global scales in calibration.++ The ``k_global_scale``/``v_global_scale`` parameters are only registered+ once the calibration forwards observed KV magnitudes, so their presence on+ all attention modules is the signal that an NVFP4 ``kv_cache_scheme`` can+ be exported with valid static scales.+ """+ from auto_round.experimental.utils import is_attention_module++ num_attention_modules = 0+ for module in model.modules():+ if is_attention_module(module):+ num_attention_modules += 1+ if not isinstance(getattr(module, "k_global_scale", None), torch.nn.Parameter) or not isinstance(+ getattr(module, "v_global_scale", None), torch.nn.Parameter+ ):+ return False+ return num_attention_modules > 0++def _build_mixed_fp_quantization_config(scheme_groups,layer_config,@@ -209,17 +232,24 @@ def _build_mixed_fp_quantization_config(group_formats[group_name] = _get_group_format(lbits, ldata_type)use_fp8_attention = _use_fp8_attention(static_attention_dtype)+ use_fp8_kv = _use_fp8_kv(static_kv_dtype)+ use_nvfp4_kv = _use_nvfp4_kv(static_kv_dtype)if use_fp8_attention:attention_config = _get_attention_config(model, static_attention_granularity)else:attention_config = Nonekv_granularity = static_attention_granularity if use_fp8_attention else static_kv_granularity+ kv_cache_scheme = _resolve_kv_cache_scheme(+ model,+ use_fp8_kv=use_fp8_kv,+ use_nvfp4_kv=use_nvfp4_kv,+ use_fp8_attention=use_fp8_attention,+ kv_granularity=kv_granularity,+ )quantization_config = initialize_quantization(scheme=None,config_groups=config_groups,- kv_cache_scheme=(- _construct_kv_scheme(kv_granularity) if (_use_fp8_kv(static_kv_dtype) or use_fp8_attention) else None- ),+ kv_cache_scheme=kv_cache_scheme,ignore=ignore,)quantization_config = quantization_config.to_dict()@@ -236,6 +266,33 @@ def _build_mixed_fp_quantization_config(return quantization_config+def _resolve_kv_cache_scheme(+ model: torch.nn.Module,+ use_fp8_kv: bool,+ use_nvfp4_kv: bool,+ use_fp8_attention: bool,+ kv_granularity: str = "tensor",+):+ """Pick the KV cache scheme for the exported compressed-tensors config."""+ if use_nvfp4_kv and (use_fp8_kv or use_fp8_attention):+ raise ValueError(+ "static_kv_dtype 'nvfp4' conflicts with the FP8 static attention/KV options; "+ "please set only one of them."+ )+ if use_nvfp4_kv:+ if not _attention_modules_have_nvfp4_kv_scales(model):+ raise ValueError(+ "NVFP4 static KV cache quantization was requested "+ "(static_kv_dtype='nvfp4') but no k_global_scale/v_global_scale "+ "parameters were collected. Quantize with calibration data and "+ "static_kv_dtype='nvfp4' set (or drop the NVFP4 KV option)."+ )+ return _construct_nvfp4_kv_scheme()+ if use_fp8_kv or use_fp8_attention:+ return _construct_kv_scheme(kv_granularity)+ return None++def save_quantized_as_fp(output_dir: str,model: torch.nn.Module = None,@@ -334,10 +391,12 @@ def save_quantized_as_fp(static_attention_granularity = serialization_dict.get("static_attention_granularity", "tensor")static_kv_granularity = serialization_dict.get("static_kv_granularity", "tensor")kv_granularity = static_attention_granularity if use_fp8_attention else static_kv_granularity- kv_cache_scheme = (- _construct_kv_scheme(kv_granularity)- if (_use_fp8_kv(serialization_dict.get("static_kv_dtype", None)) or use_fp8_attention)- else None+ kv_cache_scheme = _resolve_kv_cache_scheme(+ model,+ use_fp8_kv=_use_fp8_kv(serialization_dict.get("static_kv_dtype", None)),+ use_nvfp4_kv=_use_nvfp4_kv(serialization_dict.get("static_kv_dtype", None)),+ use_fp8_attention=use_fp8_attention,+ kv_granularity=kv_granularity,)if is_mixed:diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex a2cc52b07..4741112f5 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -23,6 +23,7 @@ import transformersfrom tqdm import tqdmfrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad+from auto_round.experimental.utils import NVFP4_KV_DTYPEfrom auto_round.export.export_to_autoround.export_to_fp8 import FP8QLinearfrom auto_round.export.export_to_llmcompressor.config import check_compressed_tensors_supportedfrom auto_round.export.utils import is_immediate_saving_mode, save_model, save_pretrained_artifact@@ -114,6 +115,32 @@ def _use_fp8_attention(static_attention_dtype: str | None) -> bool:return False+def _use_nvfp4_kv(static_kv_dtype: str | None) -> bool:+ """Return True if static KV cache should use NVFP4."""+ if static_kv_dtype in (NVFP4_KV_DTYPE, "fp4", "float4_e2m1"):+ logger.warning_once("Exporting model with static KV cache in NVFP4 dtype.")+ return True+ return False+++def _construct_nvfp4_kv_scheme():+ """Construct the NVFP4 KV cache scheme from the compressed-tensors NVFP4 preset.++ Reusing the preset's input-activation args guarantees the serialized+ ``kv_cache_scheme`` is identical to what llm-compressor / compressed-tensors+ produce for NVFP4 KV cache (fp4 e2m1, 16-element groups, fp8_e4m3 block+ scales computed at runtime, static per-tensor global scale).+ """+ from compressed_tensors.quantization import preset_name_to_scheme # pylint: disable=E0401++ kv_args = preset_name_to_scheme("NVFP4", ["Linear"]).input_activations+ logger.warning_once(+ "Using the compressed-tensors NVFP4 preset input-activation args as the KV cache scheme: %s",+ repr(kv_args),+ )+ return kv_args++def _get_attention_targets(model: torch.nn.Module) -> list[str]:from compressed_tensors.quantization.lifecycle.initialize import is_attention_module # pylint: disable=E0401@@ -248,6 +275,11 @@ def save_quantized_as_static_fp(static_kv_granularity = serialization_dict.get("static_kv_granularity", "tensor")use_fp8_attention = _use_fp8_attention(serialization_dict.get("static_attention_dtype", None))use_fp8_kv = _use_fp8_kv(serialization_dict.get("static_kv_dtype", None))+ if _use_nvfp4_kv(serialization_dict.get("static_kv_dtype", None)):+ raise ValueError(+ "NVFP4 KV cache export is only supported on the NVFP4 compressed-tensors export path "+ "(NVFP4 weight scheme + format 'llm_compressor'); this path (FP8_STATIC) supports FP8 KV cache only."+ )attention_config = _get_attention_config(model, static_attention_granularity) if use_fp8_attention else Nonekv_granularity = static_attention_granularity if use_fp8_attention else static_kv_granularityquantization_config = QuantizationConfig(diff --git a/test/unit/test_cpu/calibration/__init__.py b/test/unit/test_cpu/calibration/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py b/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.pynew file mode 100644index 000000000..535c6273a--- /dev/null+++ b/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py@@ -0,0 +1,127 @@+"""Regression test for QuantizedKVParameterCache.update (NVFP4 path).++A previous revision passed ``key_states`` to both the KEY and the VALUE+``_nvfp4_quant_dequant`` calls, so ``v_amax`` was a copy of ``k_amax`` and+the exported ``v_global_scale`` silently equaled ``k_global_scale`` for+every layer. The cache must track K and V magnitudes separately and QDQ+each side from its own tensor.+"""++import pytest+import torch++from auto_round.experimental.kv_cache import QuantizedKVParameterCache+++def _reset_singleton():+ # ``reset`` is an instance method; clear the class state directly.+ QuantizedKVParameterCache._instance = None+ QuantizedKVParameterCache._initialized = False++[email protected](autouse=True)+def _fresh_cache():+ _reset_singleton()+ yield+ _reset_singleton()+++def test_update_tracks_k_and_v_amax_separately():+ cache = QuantizedKVParameterCache(dtype="nvfp4")+ assert cache.is_nvfp4++ torch.manual_seed(0)+ # Deliberately different magnitudes per side so a K/V mixup cannot hide.+ key_states = torch.randn(2, 4, 64, 32) * 3.0+ value_states = torch.randn(2, 4, 64, 32) * 11.0+ k_amax = key_states.abs().max().item()+ v_amax = value_states.abs().max().item()+ assert v_amax > 2 * k_amax # sanity: the two sides are distinguishable++ qk, qv = cache.update(key_states, value_states, layer_idx=0)++ assert cache.k_amax[0] == pytest.approx(k_amax)+ assert cache.v_amax[0] == pytest.approx(v_amax)++ # Calibration is observe-only: K/V pass through unmodified; the exported+ # static scale is derived from the final amax (see the convention test+ # below). This mirrors llm-compressor's observer-based KV calibration.+ assert torch.equal(qk, key_states)+ assert torch.equal(qv, value_states)+++def test_update_running_max_per_side():+ cache = QuantizedKVParameterCache(dtype="nvfp4")++ k_small = torch.full((1, 1, 16, 16), 1.0)+ v_big = torch.full((1, 1, 16, 16), 8.0)+ cache.update(k_small, v_big, layer_idx=2)+ assert cache.k_amax[2] == pytest.approx(1.0)+ assert cache.v_amax[2] == pytest.approx(8.0)++ # A later larger K must not leak into V, and vice versa.+ k_big = torch.full((1, 1, 16, 16), 16.0)+ v_small = torch.full((1, 1, 16, 16), 0.5)+ cache.update(k_big, v_small, layer_idx=2)+ assert cache.k_amax[2] == pytest.approx(16.0)+ assert cache.v_amax[2] == pytest.approx(8.0)+++def test_stored_nvfp4_scale_uses_vllm_dequant_multiplier_convention():+ """The checkpoint scale must be amax / 2688 (vLLM dequant multiplier).++ vLLM's store kernel computes ``global_scale = 1 / k_scale`` and+ ``sf = global_scale * block_max / 6``. Storing the weight-style+ ``2688 / amax`` instead makes the runtime fp8 block scale+ ``block_max * amax / (6 * 2688)`` underflow e4m3 (min subnormal 2**-9)+ for typical activation amax values, zeroing the served KV cache.+ """+ from auto_round.experimental.kv_cache import _nvfp4_global_scale++ device = torch.device("cpu")+ for amax in (0.5, 2.0, 282.0):+ scale = _nvfp4_global_scale(amax, device)+ assert scale.shape == (1,) and scale.dtype == torch.float32+ # Reciprocal of the weight-style global scale (2688 / amax).+ assert scale[0] == pytest.approx(amax / (448.0 * 6.0), rel=1e-6)+ assert 1.0 / scale[0] == pytest.approx(448.0 * 6.0 / amax, rel=1e-5)++ # Every runtime fp8 block scale, from a block at 1% of the calibrated+ # amax up to the worst-case block, must stay representable in e4m3.+ g_rt = 1.0 / scale[0]+ block_maxes = torch.linspace(0.01 * amax, amax, 8)+ sf = g_rt * block_maxes / 6.0+ assert sf.max().item() <= 448.0 + 1e-4+ # e4m3 min subnormal is 2**-9 (~1.95e-3)+ assert sf.min().item() >= 2.0**-9+++def _make_fake_attention(head_dim: int, num_heads: int = 4):+ import torch.nn as nn++ class FakeAttention(nn.Module):+ def __init__(self):+ super().__init__()+ self.layer_idx = 0+ self.head_dim = head_dim+ self.k_proj = nn.Linear(head_dim, num_heads * head_dim, bias=False)+ self.v_proj = nn.Linear(head_dim, num_heads * head_dim, bias=False)++ return FakeAttention()+++def test_nvfp4_kv_incompatible_head_dim_fails_early():+ """A single unsupported layer must fail loudly, not disable all layers."""+ from auto_round.experimental.kv_cache import initialize_quantized_kv_cache++ module = _make_fake_attention(head_dim=88)+ with pytest.raises(ValueError, match="divisible by 16"):+ initialize_quantized_kv_cache(module, dtype="nvfp4")+ assert not hasattr(module, "kv_cache")+++def test_nvfp4_kv_incompatible_head_dim_raises_at_update():+ cache = QuantizedKVParameterCache(dtype="nvfp4")+ bad = torch.randn(1, 1, 16, 88)+ with pytest.raises(ValueError, match="divisible by 16"):+ cache.update(bad, bad.clone(), layer_idx=0)diff --git a/test/unit/test_cpu/export/test_llmc_format.py b/test/unit/test_cpu/export/test_llmc_format.pyindex 3a080d38a..02411baa9 100644--- a/test/unit/test_cpu/export/test_llmc_format.py+++ b/test/unit/test_cpu/export/test_llmc_format.py@@ -452,6 +452,165 @@ class TestLLMC:), f"Invalid mixed precision quantization configuration: {quantization_config}"+class TestLLMCNVFP4KV:+ """NVFP4 static KV cache: calibration, saved global scales, and CT-parity export."""++ @staticmethod+ def _expected_nvfp4_kv_args():+ from compressed_tensors.quantization import preset_name_to_scheme++ return preset_name_to_scheme("NVFP4", ["Linear"]).input_activations.model_dump()++ def _quantize_nvfp4_kv(self, tiny_opt_model_path, dataloader, tmp_path):+ return AutoRound(+ tiny_opt_model_path,+ scheme="NVFP4",+ seqlen=8,+ nsamples=2,+ iters=0,+ dataset=dataloader,+ static_kv_dtype="nvfp4",+ ).quantize_and_save(tmp_path, format="llm_compressor")++ def test_nvfp4_kv_missing_scales_raises(self):+ """An explicit NVFP4 KV request must not silently drop the scheme."""+ import torch.nn as nn++ class FakeAttention(nn.Module):+ def __init__(self):+ super().__init__()+ self.layer_idx = 0+ self.head_dim = 64+ self.k_proj = nn.Linear(64, 256, bias=False)+ self.v_proj = nn.Linear(64, 256, bias=False)++ class FakeModel(nn.Module):+ def __init__(self):+ super().__init__()+ self.attn = FakeAttention()++ with pytest.raises(ValueError, match="NVFP4"):+ llmc_fp_export._resolve_kv_cache_scheme(+ FakeModel(),+ use_fp8_kv=False,+ use_nvfp4_kv=True,+ use_fp8_attention=False,+ )++ @pytest.mark.timeout(120)+ def test_llmcompressor_nvfp4_kv_config(self, tiny_opt_model_path, dataloader, tmp_path):+ _, quantized_model_path = self._quantize_nvfp4_kv(tiny_opt_model_path, dataloader, tmp_path)++ with open(os.path.join(quantized_model_path, "config.json")) as f:+ config = json.load(f)+ kv_cache_scheme = config["quantization_config"]["kv_cache_scheme"]+ assert kv_cache_scheme is not None+ # Must match the compressed-tensors NVFP4 preset input-activation args+ # (same convention llm-compressor serializes for NVFP4 KV cache).+ expected = self._expected_nvfp4_kv_args()+ for key, value in expected.items():+ assert (+ kv_cache_scheme.get(key) == value+ ), f"kv_cache_scheme[{key!r}] mismatch: {kv_cache_scheme.get(key)!r} != {value!r}"+ assert config["quantization_config"]["format"] == "nvfp4-pack-quantized"++ @pytest.mark.timeout(120)+ def test_llmcompressor_nvfp4_kv_global_scales(self, tiny_opt_model_path, dataloader, tmp_path):+ from safetensors import safe_open++ from auto_round.data_type.nvfp import calculate_gparam+ from auto_round.experimental.kv_cache import QuantizedKVParameterCache++ _, quantized_model_path = self._quantize_nvfp4_kv(tiny_opt_model_path, dataloader, tmp_path)++ with open(os.path.join(quantized_model_path, "config.json")) as f:+ config = json.load(f)+ num_layers = config["num_hidden_layers"]++ # Calibration must have observed KV magnitudes (data-driven path).+ cache = QuantizedKVParameterCache._instance+ assert cache is not None and cache.is_nvfp4+ assert len(cache.k_amax) >= num_layers and len(cache.v_amax) >= num_layers++ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:+ for i in range(num_layers):+ k_global_scale = f.get_tensor(f"model.decoder.layers.{i}.self_attn.k_global_scale")+ v_global_scale = f.get_tensor(f"model.decoder.layers.{i}.self_attn.v_global_scale")+ assert k_global_scale.dtype == torch.float32 and k_global_scale.shape == torch.Size([1])+ assert v_global_scale.dtype == torch.float32 and v_global_scale.shape == torch.Size([1])+ assert cache.k_amax[i] > 0 and cache.v_amax[i] > 0+ # vLLM serving convention: the checkpoint stores the dequantization+ # multiplier amax / 2688 = 1 / calculate_gparam(amax) (see+ # _nvfp4_global_scale), not the weight-style 2688 / amax.+ expected_k = 1.0 / calculate_gparam(cache.k_amax[i])+ expected_v = 1.0 / calculate_gparam(cache.v_amax[i])+ assert torch.isclose(k_global_scale.flatten()[0], expected_k, rtol=1e-5, atol=0.0)+ assert torch.isclose(v_global_scale.flatten()[0], expected_v, rtol=1e-5, atol=0.0)++ @pytest.mark.timeout(120)+ def test_nvfp4_kv_scheme_round_trip_matches_vllm_runtime(self, tiny_opt_model_path, dataloader, tmp_path):+ from safetensors import safe_open++ from auto_round.data_type.nvfp import cast_to_fp4, nv_fp4_with_static_gs+ from auto_round.experimental.kv_cache import QuantizedKVParameterCache++ # The stored scale must make the *vLLM* NVFP4 KV store/read kernels+ # reproduce the calibration-time QDQ. The pure-Python compressed-tensors+ # runtime (forward_quantize) is no longer a valid reference here: it+ # interprets the checkpoint value as the weight-style global scale+ # (sf = gs * block_max / 6), i.e. the opposite convention of vLLM+ # (sf = (1 / k_scale) * block_max / 6). vLLM is the target serving+ # engine, so this test mirrors the vLLM kernel math instead.+ _, quantized_model_path = self._quantize_nvfp4_kv(tiny_opt_model_path, dataloader, tmp_path)++ with open(os.path.join(quantized_model_path, "config.json")) as f:+ config = json.load(f)+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:+ k_global_scale = f.get_tensor("model.decoder.layers.0.self_attn.k_global_scale")+ val = k_global_scale.to(torch.float32).flatten()[0]++ torch.manual_seed(0)+ k_states = torch.randn(1, 2, 16, 64)++ # vLLM store kernel (reshape_and_cache_nvfp4, via cvt_warp_fp16_to_fp4+ # with SFScaleVal = 1/val): sf = (1/val) * block_max/6 rounded to+ # fp8_e4m3; fp4 code = x * (1/val) / sf (saturating at +/-6).+ xb = k_states.to(torch.float32).unflatten(-1, (-1, 16))+ block_max = xb.abs().amax(dim=-1, keepdim=True)+ sf8 = ((block_max / 6.0) / val).to(torch.float8_e4m3fn).to(torch.float32)+ out_scale = torch.where(sf8 == 0, torch.zeros_like(sf8), (1.0 / val) / sf8)+ fp4 = cast_to_fp4((xb * out_scale).clamp(-6.0, 6.0))+ # vLLM reader: x_hat = fp4 * sf * val+ vllm_runtime_qdq = (fp4 * sf8 * val).flatten(-2).to(k_states.dtype)+ assert not torch.allclose(vllm_runtime_qdq, k_states), "quantization had no effect"++ # Calibration-time QDQ simulation (same amax that produced `val`):+ # the fp8 block scales must be bit-identical and the dequantized+ # values may only differ by fp32 associativity noise -- far below+ # one fp4 step (amax / 3), which a wrong scale convention would+ # produce (e.g. zeroed blocks under the old 2688 / amax convention).+ cache = QuantizedKVParameterCache._instance+ assert cache is not None and cache.is_nvfp4+ sim_qdq, sim_scale, _ = nv_fp4_with_static_gs(k_states, tensor_max=cache.k_amax[0])+ assert torch.equal(sf8, sim_scale.reshape(sf8.shape)), "fp8 block scales differ from calibration"+ max_diff = (vllm_runtime_qdq - sim_qdq).abs().max().item()+ assert max_diff <= 1e-4 * k_states.abs().max().item(), f"max diff {max_diff}"++ @pytest.mark.timeout(120)+ def test_nvfp4_kv_conflicts_with_fp8_static(self, tiny_opt_model_path, dataloader, tmp_path):+ autoround = AutoRound(+ tiny_opt_model_path,+ scheme="FP8_STATIC",+ seqlen=8,+ nsamples=2,+ iters=0,+ dataset=dataloader,+ static_kv_dtype="nvfp4",+ )+ with pytest.raises(ValueError, match="NVFP4"):+ autoround.quantize_and_save(tmp_path, format="llm_compressor")++def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, dataloader, tmp_path, monkeypatch):autoround = AutoRound(tiny_opt_model_path,