Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

feat: add NVFP4 static KV cache quantization support (#2366)

Yi Liu04fca6225906
diff --git a/auto_round/cli/parser.py b/auto_round/cli/parser.py
index 77793f1b4..bad251a7a 100644
--- a/auto_round/cli/parser.py
+++ b/auto_round/cli/parser.py
@@ -244,7 +244,7 @@ def build_quantize_parser(*, prog: str = "auto_round quantize") -> argparse.Argu
"--static_kv_dtype",
default=None,
type=str,
- choices=["fp8", "float8_e4m3fn"],
+ choices=["fp8", "float8_e4m3fn", "nvfp4"],
help="Static KV-cache quantization data type.",
)
rt.add_argument(
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index fc722ef53..7beb61ccf 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -554,6 +554,12 @@ class BaseOrchestrator(object):
if isinstance(self.scheme, AutoScheme):
return True
+ # Static KV-cache / attention quantization observes KV magnitudes
+ # during the calibration forwards, so it always needs data — even for
+ # weight-only schemes (e.g. NVFP4 with iters=0).
+ if self.static_kv_dtype is not None or self.static_attention_dtype is not None:
+ return True
+
# Check if activation calibration is needed
from auto_round.compressors.utils import check_need_act_calibration
diff --git a/auto_round/experimental/kv_cache.py b/auto_round/experimental/kv_cache.py
index 5bee3e8c0..948a48f91 100644
--- a/auto_round/experimental/kv_cache.py
+++ b/auto_round/experimental/kv_cache.py
@@ -25,6 +25,9 @@ import torch
from transformers.cache_utils import DynamicCache
from auto_round.experimental.utils import (
+ FP8_GRANULARITY_TENSOR,
+ NVFP4_KV_BLOCK_SIZE,
+ NVFP4_KV_DTYPE,
fp8_qdq,
is_attention_module,
normalize_fp8_granularity,
@@ -107,16 +110,25 @@ class QuantizedKVParameterCache(DynamicCache):
cls._instance = super(QuantizedKVParameterCache, cls).__new__(cls)
return cls._instance
- def __init__(self, dtype: torch.dtype = torch.float8_e4m3fn, granularity: str = "tensor"):
-
- assert dtype == torch.float8_e4m3fn, "Only fp8_e4m3fn is supported for now."
- self.granularity = normalize_fp8_granularity(granularity)
+ def __init__(self, dtype: torch.dtype | str = torch.float8_e4m3fn, granularity: str = "tensor"):
+ dtype = normalize_static_kv_dtype(dtype)
+ self.is_nvfp4 = dtype == NVFP4_KV_DTYPE
+ if self.is_nvfp4:
+ # NVFP4 block scales are computed per 16-element group at runtime;
+ # only a static per-tensor global scale is calibrated, so "head"
+ # granularity does not apply.
+ self.granularity = FP8_GRANULARITY_TENSOR
+ else:
+ assert dtype == torch.float8_e4m3fn, "Only fp8_e4m3fn is supported for now."
+ self.granularity = normalize_fp8_granularity(granularity)
if not self._initialized:
super().__init__()
# each index corresponds to layer_idx of the attention layer
self.k_scales: List[torch.Tensor] = []
self.v_scales: List[torch.Tensor] = []
+ self.k_amax: List[float] = []
+ self.v_amax: List[float] = []
self._initialized = True
def update(
@@ -129,6 +141,13 @@ class QuantizedKVParameterCache(DynamicCache):
"""
Get the k_scale and v_scale and output the quant-dequant key_states and value_states
"""
+ if self.is_nvfp4:
+ # Observe-only: track the per-side running amax and let K/V
+ # pass through unmodified (see _nvfp4_track_amax).
+ self._nvfp4_track_amax(key_states, KVCacheScaleType.KEY, layer_idx)
+ self._nvfp4_track_amax(value_states, KVCacheScaleType.VALUE, layer_idx)
+ return key_states, value_states
+
qdq_key_states = self._quant_dequant(key_states.contiguous(), KVCacheScaleType.KEY, layer_idx)
qdq_value_states = self._quant_dequant(value_states.contiguous(), KVCacheScaleType.VALUE, layer_idx)
@@ -181,6 +200,32 @@ class QuantizedKVParameterCache(DynamicCache):
_pad_and_append_at_idx_(scales, layer_idx, scale.reshape(-1).detach())
return qdq_tensor
+ def _nvfp4_track_amax(self, tensor: torch.Tensor, kv_type: KVCacheScaleType, layer_idx: int):
+ """Track the running NVFP4 K/V amax during calibration (observe-only).
+
+ Mirrors the observer-based KV calibration in llm-compressor: the
+ per-side running amax is accumulated while the K/V tensors pass
+ through unmodified, so calibration never fake-quantizes with an
+ incomplete scale. The static global scale is derived from the
+ final (full-calibration-set) amax by the output hook, which
+ writes the vLLM dequant-multiplier convention (``amax / 2688``) --
+ see ``_nvfp4_global_scale``.
+ """
+ if tensor.shape[-1] % NVFP4_KV_BLOCK_SIZE != 0:
+ raise ValueError(
+ f"NVFP4 KV cache quantization requires the last dim (head_dim) "
+ f"to be divisible by {NVFP4_KV_BLOCK_SIZE} (got {tensor.shape[-1]} "
+ f"at layer_idx={layer_idx}); use static_kv_dtype='fp8' for "
+ f"this model."
+ )
+
+ amax = tensor.abs().max().item()
+ amax_list = self.k_amax if kv_type == KVCacheScaleType.KEY else self.v_amax
+ if layer_idx >= len(amax_list):
+ _pad_and_append_at_idx_(amax_list, layer_idx, amax)
+ else:
+ amax_list[layer_idx] = max(amax_list[layer_idx], amax)
+
def initialize_quantized_kv_cache(module: torch.nn.Module, dtype=torch.float8_e4m3fn, granularity: str = "tensor"):
"""
@@ -193,9 +238,28 @@ def initialize_quantized_kv_cache(module: torch.nn.Module, dtype=torch.float8_e4
if isinstance(existing_kv_cache, QuantizedKVParameterCache):
return
+ dtype = normalize_static_kv_dtype(dtype)
+ if dtype == NVFP4_KV_DTYPE:
+ # NVFP4 KV needs head_dim divisible by the fp4 block size; fail
+ # early per layer instead of silently degrading all layers.
+ head_dim = getattr(module, "head_dim", None)
+ if head_dim is not None and head_dim % NVFP4_KV_BLOCK_SIZE != 0:
+ raise ValueError(
+ f"NVFP4 KV cache quantization requires head_dim divisible by "
+ f"{NVFP4_KV_BLOCK_SIZE} (got {head_dim} in "
+ f"{module.__class__.__name__}, "
+ f"layer_idx={getattr(module, 'layer_idx', '?')}); "
+ f"use static_kv_dtype='fp8' for this model."
+ )
+
quantized_kv_cache = QuantizedKVParameterCache(dtype=dtype, granularity=granularity)
setattr(module, "kv_cache", quantized_kv_cache)
logger.debug(f"Initialized quantized kv_cache for {module.__class__.__name__} {getattr(module, 'layer_idx', None)}")
+ if quantized_kv_cache.is_nvfp4:
+ # Global scales are only registered once calibration has observed KV
+ # magnitudes; creating zero/placeholder scales up front would export a
+ # broken scheme when no calibration forward happens.
+ return
init_scale = torch.tensor([0.0], device=next(module.parameters()).device)
update_parameter_data(module, init_scale.clone(), KVCacheScaleType.KEY.value)
update_parameter_data(module, init_scale.clone(), KVCacheScaleType.VALUE.value)
@@ -220,11 +284,52 @@ def calibrate_kv_cache_input_hook(
return args, kwargs
+def _nvfp4_global_scale(amax: float, device: torch.device) -> torch.Tensor:
+ """Static NVFP4 KV scale for the checkpoint, in the vLLM serving convention.
+
+ vLLM's NVFP4 KV cache store kernel (``reshape_and_cache_nvfp4``) treats the
+ checkpoint value as the *dequantization multiplier*: it computes
+ ``global_scale = 1 / k_scale`` and the per-16-element fp8 block scale as
+ ``sf = global_scale * block_max / 6``, then reads back
+ ``x = fp4 * sf * k_scale``. Storing the weight-style ``2688 / amax``
+ instead would make the runtime build fp8 block scales from
+ ``block_max * amax / (6 * 2688)``, which underflows e4m3 (min subnormal
+ ~2**-9) for typical activation magnitudes and zeroes the KV cache.
+
+ The reciprocal ``amax / 2688 = 1 / calculate_gparam(amax)`` makes the
+ runtime's block scale ``(2688 / amax) * block_max / 6`` -- exactly the
+ value auto-round's QDQ simulation (``nv_fp4_with_static_gs``) uses during
+ calibration, so served outputs match calibration up to fp8 rounding.
+
+ Note: the pure-Python compressed-tensors runtime (``forward_quantize``)
+ interprets the stored value the other way around (``sf = gs * block_max /
+ 6``). The two runtimes use opposite conventions; vLLM is the target
+ serving engine, hence this convention.
+ """
+ from auto_round.data_type.nvfp import calculate_gparam
+
+ global_scale = calculate_gparam(amax, device=device)
+ return (1.0 / global_scale).reshape(1).detach()
+
+
def calibrate_kv_cache_output_hook(module: torch.nn.Module, _args: Any, _output: torch.Tensor):
"""
Hook to update k_scale and v_scale parameters when running kv_cache quantization.
"""
kv_cache = getattr(module, "kv_cache")
+ if kv_cache.is_nvfp4:
+ layer_idx = module.layer_idx
+ k_amax = kv_cache.k_amax[layer_idx] if layer_idx < len(kv_cache.k_amax) else 0.0
+ v_amax = kv_cache.v_amax[layer_idx] if layer_idx < len(kv_cache.v_amax) else 0.0
+ if k_amax > 0:
+ update_parameter_data(
+ module, _nvfp4_global_scale(k_amax, next(module.parameters()).device), "k_global_scale"
+ )
+ if v_amax > 0:
+ update_parameter_data(
+ module, _nvfp4_global_scale(v_amax, next(module.parameters()).device), "v_global_scale"
+ )
+ return
k_scale = kv_cache.k_scales[module.layer_idx]
v_scale = kv_cache.v_scales[module.layer_idx]
update_parameter_data(module, k_scale, KVCacheScaleType.KEY.value)
@@ -241,17 +346,27 @@ def prep_attention_module_for_calibration(module: torch.nn.Module):
def kvcache_quant_context(
model: torch.nn.Module, static_kv_dtype=torch.float8_e4m3fn, static_kv_granularity: str = "tensor"
):
- """Context manager for FP8 KV cache quantization operations."""
+ """Context manager for static KV cache quantization (FP8 or NVFP4) operations."""
try:
# Setup phase: Initialize KV cache for quantization
static_kv_dtype = normalize_static_kv_dtype(static_kv_dtype)
static_kv_granularity = normalize_fp8_granularity(static_kv_granularity)
- if static_kv_dtype != torch.float8_e4m3fn:
- logger.warning(f"Ignoring static kv dtype {static_kv_dtype}, only fp8_e4m3fn is supported.")
+ if static_kv_dtype == NVFP4_KV_DTYPE:
+ if static_kv_granularity != FP8_GRANULARITY_TENSOR:
+ logger.warning(
+ "NVFP4 KV cache only supports 'tensor' granularity; ignoring granularity %r.",
+ static_kv_granularity,
+ )
+ initialize_fn = partial(
+ initialize_quantized_kv_cache, dtype=NVFP4_KV_DTYPE, granularity=FP8_GRANULARITY_TENSOR
+ )
+ elif static_kv_dtype != torch.float8_e4m3fn:
+ logger.warning(f"Ignoring static kv dtype {static_kv_dtype}, only fp8_e4m3fn and nvfp4 are supported.")
else:
initialize_fn = partial(
initialize_quantized_kv_cache, dtype=static_kv_dtype, granularity=static_kv_granularity
)
+ if static_kv_dtype in (torch.float8_e4m3fn, NVFP4_KV_DTYPE):
model.apply(initialize_fn)
model.apply(prep_attention_module_for_calibration)
diff --git a/auto_round/experimental/utils.py b/auto_round/experimental/utils.py
index 2b106f1f1..2391d1798 100644
--- a/auto_round/experimental/utils.py
+++ b/auto_round/experimental/utils.py
@@ -112,8 +112,19 @@ def update_parameter_data(module: torch.nn.Module, new_val: torch.Tensor, name:
module.register_parameter(name, torch.nn.Parameter(new_val))
-def normalize_static_kv_dtype(static_kv_dtype: str | torch.dtype) -> torch.dtype:
- valid_dtype_name_lst = ["float16", "bfloat16", "fp8", "float32", "float"]
+NVFP4_KV_DTYPE = "nvfp4"
+NVFP4_KV_BLOCK_SIZE = 16
+
+
+def normalize_static_kv_dtype(static_kv_dtype: str | torch.dtype) -> torch.dtype | str:
+ """Normalize the static KV-cache quantization dtype.
+
+ Returns a ``torch.dtype`` for float dtypes, or the ``"nvfp4"`` sentinel
+ (see :data:`NVFP4_KV_DTYPE`) for NVFP4 KV cache quantization.
+ """
+ if isinstance(static_kv_dtype, torch.dtype):
+ return static_kv_dtype
+ valid_dtype_name_lst = ["float16", "bfloat16", "fp8", "float32", "float", NVFP4_KV_DTYPE, "fp4", "float4_e2m1"]
valid_torch_dtype = {
"float16": torch.float16,
"bfloat16": torch.bfloat16,
@@ -122,6 +133,8 @@ def normalize_static_kv_dtype(static_kv_dtype: str | torch.dtype) -> torch.dtype
"float32": torch.float32,
"float": torch.float32, # Alias for float32
}
+ if static_kv_dtype in (NVFP4_KV_DTYPE, "fp4", "float4_e2m1"):
+ return NVFP4_KV_DTYPE
if static_kv_dtype in valid_dtype_name_lst:
new_dtype = valid_torch_dtype[static_kv_dtype]
elif static_kv_dtype in valid_torch_dtype.values():
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index f207a035d..9746b7e46 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -54,9 +54,11 @@ from .config import check_compressed_tensors_supported
from .export_to_static_fp import (
_configure_gaudi2_fp8_dtype,
_construct_kv_scheme,
+ _construct_nvfp4_kv_scheme,
_get_attention_config,
_use_fp8_attention,
_use_fp8_kv,
+ _use_nvfp4_kv,
)
__all__ = [
@@ -156,6 +158,27 @@ def _get_group_format(bits, data_type):
return "float-quantized"
+def _attention_modules_have_nvfp4_kv_scales(model: torch.nn.Module) -> bool:
+ """True when every attention module collected NVFP4 KV global scales in calibration.
+
+ The ``k_global_scale``/``v_global_scale`` parameters are only registered
+ once the calibration forwards observed KV magnitudes, so their presence on
+ all attention modules is the signal that an NVFP4 ``kv_cache_scheme`` can
+ be exported with valid static scales.
+ """
+ from auto_round.experimental.utils import is_attention_module
+
+ num_attention_modules = 0
+ for module in model.modules():
+ if is_attention_module(module):
+ num_attention_modules += 1
+ if not isinstance(getattr(module, "k_global_scale", None), torch.nn.Parameter) or not isinstance(
+ getattr(module, "v_global_scale", None), torch.nn.Parameter
+ ):
+ return False
+ return num_attention_modules > 0
+
+
def _build_mixed_fp_quantization_config(
scheme_groups,
layer_config,
@@ -209,17 +232,24 @@ def _build_mixed_fp_quantization_config(
group_formats[group_name] = _get_group_format(lbits, ldata_type)
use_fp8_attention = _use_fp8_attention(static_attention_dtype)
+ use_fp8_kv = _use_fp8_kv(static_kv_dtype)
+ use_nvfp4_kv = _use_nvfp4_kv(static_kv_dtype)
if use_fp8_attention:
attention_config = _get_attention_config(model, static_attention_granularity)
else:
attention_config = None
kv_granularity = static_attention_granularity if use_fp8_attention else static_kv_granularity
+ kv_cache_scheme = _resolve_kv_cache_scheme(
+ model,
+ use_fp8_kv=use_fp8_kv,
+ use_nvfp4_kv=use_nvfp4_kv,
+ use_fp8_attention=use_fp8_attention,
+ kv_granularity=kv_granularity,
+ )
quantization_config = initialize_quantization(
scheme=None,
config_groups=config_groups,
- kv_cache_scheme=(
- _construct_kv_scheme(kv_granularity) if (_use_fp8_kv(static_kv_dtype) or use_fp8_attention) else None
- ),
+ kv_cache_scheme=kv_cache_scheme,
ignore=ignore,
)
quantization_config = quantization_config.to_dict()
@@ -236,6 +266,33 @@ def _build_mixed_fp_quantization_config(
return quantization_config
+def _resolve_kv_cache_scheme(
+ model: torch.nn.Module,
+ use_fp8_kv: bool,
+ use_nvfp4_kv: bool,
+ use_fp8_attention: bool,
+ kv_granularity: str = "tensor",
+):
+ """Pick the KV cache scheme for the exported compressed-tensors config."""
+ if use_nvfp4_kv and (use_fp8_kv or use_fp8_attention):
+ raise ValueError(
+ "static_kv_dtype 'nvfp4' conflicts with the FP8 static attention/KV options; "
+ "please set only one of them."
+ )
+ if use_nvfp4_kv:
+ if not _attention_modules_have_nvfp4_kv_scales(model):
+ raise ValueError(
+ "NVFP4 static KV cache quantization was requested "
+ "(static_kv_dtype='nvfp4') but no k_global_scale/v_global_scale "
+ "parameters were collected. Quantize with calibration data and "
+ "static_kv_dtype='nvfp4' set (or drop the NVFP4 KV option)."
+ )
+ return _construct_nvfp4_kv_scheme()
+ if use_fp8_kv or use_fp8_attention:
+ return _construct_kv_scheme(kv_granularity)
+ return None
+
+
def save_quantized_as_fp(
output_dir: str,
model: torch.nn.Module = None,
@@ -334,10 +391,12 @@ def save_quantized_as_fp(
static_attention_granularity = serialization_dict.get("static_attention_granularity", "tensor")
static_kv_granularity = serialization_dict.get("static_kv_granularity", "tensor")
kv_granularity = static_attention_granularity if use_fp8_attention else static_kv_granularity
- kv_cache_scheme = (
- _construct_kv_scheme(kv_granularity)
- if (_use_fp8_kv(serialization_dict.get("static_kv_dtype", None)) or use_fp8_attention)
- else None
+ kv_cache_scheme = _resolve_kv_cache_scheme(
+ model,
+ use_fp8_kv=_use_fp8_kv(serialization_dict.get("static_kv_dtype", None)),
+ use_nvfp4_kv=_use_nvfp4_kv(serialization_dict.get("static_kv_dtype", None)),
+ use_fp8_attention=use_fp8_attention,
+ kv_granularity=kv_granularity,
)
if is_mixed:
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index a2cc52b07..4741112f5 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -23,6 +23,7 @@ import transformers
from tqdm import tqdm
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad
+from auto_round.experimental.utils import NVFP4_KV_DTYPE
from auto_round.export.export_to_autoround.export_to_fp8 import FP8QLinear
from auto_round.export.export_to_llmcompressor.config import check_compressed_tensors_supported
from auto_round.export.utils import is_immediate_saving_mode, save_model, save_pretrained_artifact
@@ -114,6 +115,32 @@ def _use_fp8_attention(static_attention_dtype: str | None) -> bool:
return False
+def _use_nvfp4_kv(static_kv_dtype: str | None) -> bool:
+ """Return True if static KV cache should use NVFP4."""
+ if static_kv_dtype in (NVFP4_KV_DTYPE, "fp4", "float4_e2m1"):
+ logger.warning_once("Exporting model with static KV cache in NVFP4 dtype.")
+ return True
+ return False
+
+
+def _construct_nvfp4_kv_scheme():
+ """Construct the NVFP4 KV cache scheme from the compressed-tensors NVFP4 preset.
+
+ Reusing the preset's input-activation args guarantees the serialized
+ ``kv_cache_scheme`` is identical to what llm-compressor / compressed-tensors
+ produce for NVFP4 KV cache (fp4 e2m1, 16-element groups, fp8_e4m3 block
+ scales computed at runtime, static per-tensor global scale).
+ """
+ from compressed_tensors.quantization import preset_name_to_scheme # pylint: disable=E0401
+
+ kv_args = preset_name_to_scheme("NVFP4", ["Linear"]).input_activations
+ logger.warning_once(
+ "Using the compressed-tensors NVFP4 preset input-activation args as the KV cache scheme: %s",
+ repr(kv_args),
+ )
+ return kv_args
+
+
def _get_attention_targets(model: torch.nn.Module) -> list[str]:
from compressed_tensors.quantization.lifecycle.initialize import is_attention_module # pylint: disable=E0401
@@ -248,6 +275,11 @@ def save_quantized_as_static_fp(
static_kv_granularity = serialization_dict.get("static_kv_granularity", "tensor")
use_fp8_attention = _use_fp8_attention(serialization_dict.get("static_attention_dtype", None))
use_fp8_kv = _use_fp8_kv(serialization_dict.get("static_kv_dtype", None))
+ if _use_nvfp4_kv(serialization_dict.get("static_kv_dtype", None)):
+ raise ValueError(
+ "NVFP4 KV cache export is only supported on the NVFP4 compressed-tensors export path "
+ "(NVFP4 weight scheme + format 'llm_compressor'); this path (FP8_STATIC) supports FP8 KV cache only."
+ )
attention_config = _get_attention_config(model, static_attention_granularity) if use_fp8_attention else None
kv_granularity = static_attention_granularity if use_fp8_attention else static_kv_granularity
quantization_config = QuantizationConfig(
diff --git a/test/unit/test_cpu/calibration/__init__.py b/test/unit/test_cpu/calibration/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py b/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py
new file mode 100644
index 000000000..535c6273a
--- /dev/null
+++ b/test/unit/test_cpu/calibration/test_kv_cache_nvfp4_update.py
@@ -0,0 +1,127 @@
+"""Regression test for QuantizedKVParameterCache.update (NVFP4 path).
+
+A previous revision passed ``key_states`` to both the KEY and the VALUE
+``_nvfp4_quant_dequant`` calls, so ``v_amax`` was a copy of ``k_amax`` and
+the exported ``v_global_scale`` silently equaled ``k_global_scale`` for
+every layer. The cache must track K and V magnitudes separately and QDQ
+each side from its own tensor.
+"""
+
+import pytest
+import torch
+
+from auto_round.experimental.kv_cache import QuantizedKVParameterCache
+
+
+def _reset_singleton():
+ # ``reset`` is an instance method; clear the class state directly.
+ QuantizedKVParameterCache._instance = None
+ QuantizedKVParameterCache._initialized = False
+
+
[email protected](autouse=True)
+def _fresh_cache():
+ _reset_singleton()
+ yield
+ _reset_singleton()
+
+
+def test_update_tracks_k_and_v_amax_separately():
+ cache = QuantizedKVParameterCache(dtype="nvfp4")
+ assert cache.is_nvfp4
+
+ torch.manual_seed(0)
+ # Deliberately different magnitudes per side so a K/V mixup cannot hide.
+ key_states = torch.randn(2, 4, 64, 32) * 3.0
+ value_states = torch.randn(2, 4, 64, 32) * 11.0
+ k_amax = key_states.abs().max().item()
+ v_amax = value_states.abs().max().item()
+ assert v_amax > 2 * k_amax # sanity: the two sides are distinguishable
+
+ qk, qv = cache.update(key_states, value_states, layer_idx=0)
+
+ assert cache.k_amax[0] == pytest.approx(k_amax)
+ assert cache.v_amax[0] == pytest.approx(v_amax)
+
+ # Calibration is observe-only: K/V pass through unmodified; the exported
+ # static scale is derived from the final amax (see the convention test
+ # below). This mirrors llm-compressor's observer-based KV calibration.
+ assert torch.equal(qk, key_states)
+ assert torch.equal(qv, value_states)
+
+
+def test_update_running_max_per_side():
+ cache = QuantizedKVParameterCache(dtype="nvfp4")
+
+ k_small = torch.full((1, 1, 16, 16), 1.0)
+ v_big = torch.full((1, 1, 16, 16), 8.0)
+ cache.update(k_small, v_big, layer_idx=2)
+ assert cache.k_amax[2] == pytest.approx(1.0)
+ assert cache.v_amax[2] == pytest.approx(8.0)
+
+ # A later larger K must not leak into V, and vice versa.
+ k_big = torch.full((1, 1, 16, 16), 16.0)
+ v_small = torch.full((1, 1, 16, 16), 0.5)
+ cache.update(k_big, v_small, layer_idx=2)
+ assert cache.k_amax[2] == pytest.approx(16.0)
+ assert cache.v_amax[2] == pytest.approx(8.0)
+
+
+def test_stored_nvfp4_scale_uses_vllm_dequant_multiplier_convention():
+ """The checkpoint scale must be amax / 2688 (vLLM dequant multiplier).
+
+ vLLM's store kernel computes ``global_scale = 1 / k_scale`` and
+ ``sf = global_scale * block_max / 6``. Storing the weight-style
+ ``2688 / amax`` instead makes the runtime fp8 block scale
+ ``block_max * amax / (6 * 2688)`` underflow e4m3 (min subnormal 2**-9)
+ for typical activation amax values, zeroing the served KV cache.
+ """
+ from auto_round.experimental.kv_cache import _nvfp4_global_scale
+
+ device = torch.device("cpu")
+ for amax in (0.5, 2.0, 282.0):
+ scale = _nvfp4_global_scale(amax, device)
+ assert scale.shape == (1,) and scale.dtype == torch.float32
+ # Reciprocal of the weight-style global scale (2688 / amax).
+ assert scale[0] == pytest.approx(amax / (448.0 * 6.0), rel=1e-6)
+ assert 1.0 / scale[0] == pytest.approx(448.0 * 6.0 / amax, rel=1e-5)
+
+ # Every runtime fp8 block scale, from a block at 1% of the calibrated
+ # amax up to the worst-case block, must stay representable in e4m3.
+ g_rt = 1.0 / scale[0]
+ block_maxes = torch.linspace(0.01 * amax, amax, 8)
+ sf = g_rt * block_maxes / 6.0
+ assert sf.max().item() <= 448.0 + 1e-4
+ # e4m3 min subnormal is 2**-9 (~1.95e-3)
+ assert sf.min().item() >= 2.0**-9
+
+
+def _make_fake_attention(head_dim: int, num_heads: int = 4):
+ import torch.nn as nn
+
+ class FakeAttention(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer_idx = 0
+ self.head_dim = head_dim
+ self.k_proj = nn.Linear(head_dim, num_heads * head_dim, bias=False)
+ self.v_proj = nn.Linear(head_dim, num_heads * head_dim, bias=False)
+
+ return FakeAttention()
+
+
+def test_nvfp4_kv_incompatible_head_dim_fails_early():
+ """A single unsupported layer must fail loudly, not disable all layers."""
+ from auto_round.experimental.kv_cache import initialize_quantized_kv_cache
+
+ module = _make_fake_attention(head_dim=88)
+ with pytest.raises(ValueError, match="divisible by 16"):
+ initialize_quantized_kv_cache(module, dtype="nvfp4")
+ assert not hasattr(module, "kv_cache")
+
+
+def test_nvfp4_kv_incompatible_head_dim_raises_at_update():
+ cache = QuantizedKVParameterCache(dtype="nvfp4")
+ bad = torch.randn(1, 1, 16, 88)
+ with pytest.raises(ValueError, match="divisible by 16"):
+ cache.update(bad, bad.clone(), layer_idx=0)
diff --git a/test/unit/test_cpu/export/test_llmc_format.py b/test/unit/test_cpu/export/test_llmc_format.py
index 3a080d38a..02411baa9 100644
--- a/test/unit/test_cpu/export/test_llmc_format.py
+++ b/test/unit/test_cpu/export/test_llmc_format.py
@@ -452,6 +452,165 @@ class TestLLMC:
), f"Invalid mixed precision quantization configuration: {quantization_config}"
+class TestLLMCNVFP4KV:
+ """NVFP4 static KV cache: calibration, saved global scales, and CT-parity export."""
+
+ @staticmethod
+ def _expected_nvfp4_kv_args():
+ from compressed_tensors.quantization import preset_name_to_scheme
+
+ return preset_name_to_scheme("NVFP4", ["Linear"]).input_activations.model_dump()
+
+ def _quantize_nvfp4_kv(self, tiny_opt_model_path, dataloader, tmp_path):
+ return AutoRound(
+ tiny_opt_model_path,
+ scheme="NVFP4",
+ seqlen=8,
+ nsamples=2,
+ iters=0,
+ dataset=dataloader,
+ static_kv_dtype="nvfp4",
+ ).quantize_and_save(tmp_path, format="llm_compressor")
+
+ def test_nvfp4_kv_missing_scales_raises(self):
+ """An explicit NVFP4 KV request must not silently drop the scheme."""
+ import torch.nn as nn
+
+ class FakeAttention(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer_idx = 0
+ self.head_dim = 64
+ self.k_proj = nn.Linear(64, 256, bias=False)
+ self.v_proj = nn.Linear(64, 256, bias=False)
+
+ class FakeModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.attn = FakeAttention()
+
+ with pytest.raises(ValueError, match="NVFP4"):
+ llmc_fp_export._resolve_kv_cache_scheme(
+ FakeModel(),
+ use_fp8_kv=False,
+ use_nvfp4_kv=True,
+ use_fp8_attention=False,
+ )
+
+ @pytest.mark.timeout(120)
+ def test_llmcompressor_nvfp4_kv_config(self, tiny_opt_model_path, dataloader, tmp_path):
+ _, quantized_model_path = self._quantize_nvfp4_kv(tiny_opt_model_path, dataloader, tmp_path)
+
+ with open(os.path.join(quantized_model_path, "config.json")) as f:
+ config = json.load(f)
+ kv_cache_scheme = config["quantization_config"]["kv_cache_scheme"]
+ assert kv_cache_scheme is not None
+ # Must match the compressed-tensors NVFP4 preset input-activation args
+ # (same convention llm-compressor serializes for NVFP4 KV cache).
+ expected = self._expected_nvfp4_kv_args()
+ for key, value in expected.items():
+ assert (
+ kv_cache_scheme.get(key) == value
+ ), f"kv_cache_scheme[{key!r}] mismatch: {kv_cache_scheme.get(key)!r} != {value!r}"
+ assert config["quantization_config"]["format"] == "nvfp4-pack-quantized"
+
+ @pytest.mark.timeout(120)
+ def test_llmcompressor_nvfp4_kv_global_scales(self, tiny_opt_model_path, dataloader, tmp_path):
+ from safetensors import safe_open
+
+ from auto_round.data_type.nvfp import calculate_gparam
+ from auto_round.experimental.kv_cache import QuantizedKVParameterCache
+
+ _, quantized_model_path = self._quantize_nvfp4_kv(tiny_opt_model_path, dataloader, tmp_path)
+
+ with open(os.path.join(quantized_model_path, "config.json")) as f:
+ config = json.load(f)
+ num_layers = config["num_hidden_layers"]
+
+ # Calibration must have observed KV magnitudes (data-driven path).
+ cache = QuantizedKVParameterCache._instance
+ assert cache is not None and cache.is_nvfp4
+ assert len(cache.k_amax) >= num_layers and len(cache.v_amax) >= num_layers
+
+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:
+ for i in range(num_layers):
+ k_global_scale = f.get_tensor(f"model.decoder.layers.{i}.self_attn.k_global_scale")
+ v_global_scale = f.get_tensor(f"model.decoder.layers.{i}.self_attn.v_global_scale")
+ assert k_global_scale.dtype == torch.float32 and k_global_scale.shape == torch.Size([1])
+ assert v_global_scale.dtype == torch.float32 and v_global_scale.shape == torch.Size([1])
+ assert cache.k_amax[i] > 0 and cache.v_amax[i] > 0
+ # vLLM serving convention: the checkpoint stores the dequantization
+ # multiplier amax / 2688 = 1 / calculate_gparam(amax) (see
+ # _nvfp4_global_scale), not the weight-style 2688 / amax.
+ expected_k = 1.0 / calculate_gparam(cache.k_amax[i])
+ expected_v = 1.0 / calculate_gparam(cache.v_amax[i])
+ assert torch.isclose(k_global_scale.flatten()[0], expected_k, rtol=1e-5, atol=0.0)
+ assert torch.isclose(v_global_scale.flatten()[0], expected_v, rtol=1e-5, atol=0.0)
+
+ @pytest.mark.timeout(120)
+ def test_nvfp4_kv_scheme_round_trip_matches_vllm_runtime(self, tiny_opt_model_path, dataloader, tmp_path):
+ from safetensors import safe_open
+
+ from auto_round.data_type.nvfp import cast_to_fp4, nv_fp4_with_static_gs
+ from auto_round.experimental.kv_cache import QuantizedKVParameterCache
+
+ # The stored scale must make the *vLLM* NVFP4 KV store/read kernels
+ # reproduce the calibration-time QDQ. The pure-Python compressed-tensors
+ # runtime (forward_quantize) is no longer a valid reference here: it
+ # interprets the checkpoint value as the weight-style global scale
+ # (sf = gs * block_max / 6), i.e. the opposite convention of vLLM
+ # (sf = (1 / k_scale) * block_max / 6). vLLM is the target serving
+ # engine, so this test mirrors the vLLM kernel math instead.
+ _, quantized_model_path = self._quantize_nvfp4_kv(tiny_opt_model_path, dataloader, tmp_path)
+
+ with open(os.path.join(quantized_model_path, "config.json")) as f:
+ config = json.load(f)
+ with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:
+ k_global_scale = f.get_tensor("model.decoder.layers.0.self_attn.k_global_scale")
+ val = k_global_scale.to(torch.float32).flatten()[0]
+
+ torch.manual_seed(0)
+ k_states = torch.randn(1, 2, 16, 64)
+
+ # vLLM store kernel (reshape_and_cache_nvfp4, via cvt_warp_fp16_to_fp4
+ # with SFScaleVal = 1/val): sf = (1/val) * block_max/6 rounded to
+ # fp8_e4m3; fp4 code = x * (1/val) / sf (saturating at +/-6).
+ xb = k_states.to(torch.float32).unflatten(-1, (-1, 16))
+ block_max = xb.abs().amax(dim=-1, keepdim=True)
+ sf8 = ((block_max / 6.0) / val).to(torch.float8_e4m3fn).to(torch.float32)
+ out_scale = torch.where(sf8 == 0, torch.zeros_like(sf8), (1.0 / val) / sf8)
+ fp4 = cast_to_fp4((xb * out_scale).clamp(-6.0, 6.0))
+ # vLLM reader: x_hat = fp4 * sf * val
+ vllm_runtime_qdq = (fp4 * sf8 * val).flatten(-2).to(k_states.dtype)
+ assert not torch.allclose(vllm_runtime_qdq, k_states), "quantization had no effect"
+
+ # Calibration-time QDQ simulation (same amax that produced `val`):
+ # the fp8 block scales must be bit-identical and the dequantized
+ # values may only differ by fp32 associativity noise -- far below
+ # one fp4 step (amax / 3), which a wrong scale convention would
+ # produce (e.g. zeroed blocks under the old 2688 / amax convention).
+ cache = QuantizedKVParameterCache._instance
+ assert cache is not None and cache.is_nvfp4
+ sim_qdq, sim_scale, _ = nv_fp4_with_static_gs(k_states, tensor_max=cache.k_amax[0])
+ assert torch.equal(sf8, sim_scale.reshape(sf8.shape)), "fp8 block scales differ from calibration"
+ max_diff = (vllm_runtime_qdq - sim_qdq).abs().max().item()
+ assert max_diff <= 1e-4 * k_states.abs().max().item(), f"max diff {max_diff}"
+
+ @pytest.mark.timeout(120)
+ def test_nvfp4_kv_conflicts_with_fp8_static(self, tiny_opt_model_path, dataloader, tmp_path):
+ autoround = AutoRound(
+ tiny_opt_model_path,
+ scheme="FP8_STATIC",
+ seqlen=8,
+ nsamples=2,
+ iters=0,
+ dataset=dataloader,
+ static_kv_dtype="nvfp4",
+ )
+ with pytest.raises(ValueError, match="NVFP4"):
+ autoround.quantize_and_save(tmp_path, format="llm_compressor")
+
+
def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, dataloader, tmp_path, monkeypatch):
autoround = AutoRound(
tiny_opt_model_path,