PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
"""Regression test for QuantizedKVParameterCache.update (NVFP4 path).
A previous revision passed ``key_states`` to both the KEY and the VALUE
``_nvfp4_quant_dequant`` calls, so ``v_amax`` was a copy of ``k_amax`` and
the exported ``v_global_scale`` silently equaled ``k_global_scale`` for
every layer. The cache must track K and V magnitudes separately and QDQ
each side from its own tensor.
"""
import pytest
import torch
from auto_round.experimental.kv_cache import QuantizedKVParameterCache
def _reset_singleton():
QuantizedKVParameterCache._instance = None
QuantizedKVParameterCache._initialized = False
@pytest.fixture(autouse=True)
def _fresh_cache():
_reset_singleton()
yield
_reset_singleton()
def test_update_tracks_k_and_v_amax_separately():
cache = QuantizedKVParameterCache(dtype="nvfp4")
assert cache.is_nvfp4
torch.manual_seed(0)
# Deliberately different magnitudes per side so a K/V mixup cannot hide.
key_states = torch.randn(2, 4, 64, 32) * 3.0
value_states = torch.randn(2, 4, 64, 32) * 11.0
k_amax = key_states.abs().max().item()
v_amax = value_states.abs().max().item()
assert v_amax > 2 * k_amax # sanity: the two sides are distinguishable
qk, qv = cache.update(key_states, value_states, layer_idx=0)
assert cache.k_amax[0] == pytest.approx(k_amax)
assert cache.v_amax[0] == pytest.approx(v_amax)
# Calibration is observe-only: K/V pass through unmodified; the exported
# static scale is derived from the final amax (see the convention test
# below). This mirrors llm-compressor's observer-based KV calibration.
assert torch.equal(qk, key_states)
assert torch.equal(qv, value_states)
def test_update_running_max_per_side():
cache = QuantizedKVParameterCache(dtype="nvfp4")
k_small = torch.full((1, 1, 16, 16), 1.0)
v_big = torch.full((1, 1, 16, 16), 8.0)
cache.update(k_small, v_big, layer_idx=2)
assert cache.k_amax[2] == pytest.approx(1.0)
assert cache.v_amax[2] == pytest.approx(8.0)
# A later larger K must not leak into V, and vice versa.
k_big = torch.full((1, 1, 16, 16), 16.0)
v_small = torch.full((1, 1, 16, 16), 0.5)
cache.update(k_big, v_small, layer_idx=2)
assert cache.k_amax[2] == pytest.approx(16.0)
assert cache.v_amax[2] == pytest.approx(8.0)
def test_stored_nvfp4_scale_uses_vllm_dequant_multiplier_convention():
"""The checkpoint scale must be amax / 2688 (vLLM dequant multiplier).
vLLM's store kernel computes ``global_scale = 1 / k_scale`` and
``sf = global_scale * block_max / 6``. Storing the weight-style
``2688 / amax`` instead makes the runtime fp8 block scale
``block_max * amax / (6 * 2688)`` underflow e4m3 (min subnormal 2**-9)
for typical activation amax values, zeroing the served KV cache.
"""
from auto_round.experimental.kv_cache import _nvfp4_global_scale
device = torch.device("cpu")
for amax in (0.5, 2.0, 282.0):
scale = _nvfp4_global_scale(amax, device)
assert scale.shape == (1,) and scale.dtype == torch.float32
# Reciprocal of the weight-style global scale (2688 / amax).
assert scale[0] == pytest.approx(amax / (448.0 * 6.0), rel=1e-6)
assert 1.0 / scale[0] == pytest.approx(448.0 * 6.0 / amax, rel=1e-5)
# Every runtime fp8 block scale, from a block at 1% of the calibrated
# amax up to the worst-case block, must stay representable in e4m3.
g_rt = 1.0 / scale[0]
block_maxes = torch.linspace(0.01 * amax, amax, 8)
sf = g_rt * block_maxes / 6.0
assert sf.max().item() <= 448.0 + 1e-4
# e4m3 min subnormal is 2**-9 (~1.95e-3)
assert sf.min().item() >= 2.0**-9
def _make_fake_attention(head_dim: int, num_heads: int = 4):
import torch.nn as nn
class FakeAttention(nn.Module):
def __init__(self):
super().__init__()
self.layer_idx = 0
self.head_dim = head_dim
self.k_proj = nn.Linear(head_dim, num_heads * head_dim, bias=False)
self.v_proj = nn.Linear(head_dim, num_heads * head_dim, bias=False)
return FakeAttention()
def test_nvfp4_kv_incompatible_head_dim_fails_early():
"""A single unsupported layer must fail loudly, not disable all layers."""
from auto_round.experimental.kv_cache import initialize_quantized_kv_cache
module = _make_fake_attention(head_dim=88)
with pytest.raises(ValueError, match="divisible by 16"):
initialize_quantized_kv_cache(module, dtype="nvfp4")
assert not hasattr(module, "kv_cache")
def test_nvfp4_kv_incompatible_head_dim_raises_at_update():
cache = QuantizedKVParameterCache(dtype="nvfp4")
bad = torch.randn(1, 1, 16, 88)
with pytest.raises(ValueError, match="divisible by 16"):
cache.update(bad, bad.clone(), layer_idx=0)
class ObservedAttention(torch.nn.Module):
def __init__(self):
super().__init__()
self.layer_idx = 0
self.head_dim = 16
self.k_proj = torch.nn.Linear(16, 16, bias=False)
self.v_proj = torch.nn.Linear(16, 16, bias=False)
def forward(self, hidden_states, past_key_values=None, use_cache=False):
key_states = self.k_proj(hidden_states).unsqueeze(1)
value_states = self.v_proj(hidden_states).unsqueeze(1)
if past_key_values is not None:
key_states, value_states = past_key_values.update(key_states, value_states, self.layer_idx)
query_states = hidden_states.unsqueeze(1)
return torch.nn.functional.scaled_dot_product_attention(query_states, key_states, value_states)
def test_nvfp4_observer_preserves_optimization_trajectory():
import copy
from contextlib import nullcontext
from auto_round.experimental.kv_cache import kvcache_quant_context
torch.manual_seed(42)
baseline = ObservedAttention()
observed = copy.deepcopy(baseline)
inputs = [torch.randn(2, 4, 16) for _ in range(3)]
targets = [torch.randn(2, 1, 4, 16) for _ in inputs]
trajectories = []
for model, context in ((baseline, nullcontext()), (observed, kvcache_quant_context(observed, "nvfp4"))):
trajectory = []
parameters = [model.k_proj.weight, model.v_proj.weight]
with context:
for hidden_states, target in zip(inputs, targets):
model.zero_grad(set_to_none=True)
output = model(hidden_states, past_key_values=None)
loss = torch.nn.functional.mse_loss(output, target)
loss.backward()
gradients = [parameter.grad.detach().clone() for parameter in parameters]
with torch.no_grad():
for parameter in parameters:
parameter.sub_(0.01 * parameter.grad.sign())
trajectory.append((output.detach(), loss.detach(), gradients))
trajectories.append(trajectory)
for expected, actual in zip(*trajectories):
torch.testing.assert_close(actual[0], expected[0])
torch.testing.assert_close(actual[1], expected[1])
for expected_gradient, actual_gradient in zip(expected[2], actual[2]):
torch.testing.assert_close(actual_gradient, expected_gradient)
torch.testing.assert_close(actual_gradient.sign(), expected_gradient.sign(), rtol=0, atol=0)
for name, expected in baseline.named_parameters():
torch.testing.assert_close(observed.get_parameter(name), expected, rtol=0, atol=0)
@pytest.mark.parametrize("dtype", ["nvfp4", "fp8"])
def test_kv_context_removes_its_hooks(dtype):
from auto_round.experimental.kv_cache import kvcache_quant_context
model = ObservedAttention()
inputs = torch.randn(1, 4, 16)
expected = model(inputs, past_key_values=None)
with kvcache_quant_context(model, dtype):
model(inputs, past_key_values=None)
actual = model(inputs, past_key_values=None)
torch.testing.assert_close(actual, expected)
def test_kv_context_does_not_reuse_previous_amax():
from auto_round.experimental.kv_cache import kvcache_quant_context
model = ObservedAttention()
with kvcache_quant_context(model, "nvfp4"):
model.kv_cache.update(torch.full((1, 1, 1, 16), 100.0), torch.ones(1, 1, 1, 16), 0)
with kvcache_quant_context(model, "nvfp4"):
model.kv_cache.update(torch.ones(1, 1, 1, 16), torch.ones(1, 1, 1, 16), 0)
assert model.kv_cache.k_amax[0] == 1.0
def test_kv_context_cleans_up_after_exception():
from auto_round.experimental.kv_cache import kvcache_quant_context
model = ObservedAttention()
with pytest.raises(RuntimeError, match="calibration failed"), kvcache_quant_context(model, "nvfp4"):
raise RuntimeError("calibration failed")
assert not hasattr(model, "kv_cache")
model(torch.randn(1, 4, 16), past_key_values=None)
def test_kv_context_preserves_unrelated_hooks():
from auto_round.experimental.kv_cache import kvcache_quant_context
model = ObservedAttention()
calls = []
handle = model.register_forward_hook(lambda module, args, output: calls.append(True))
with kvcache_quant_context(model, "nvfp4"):
model(torch.randn(1, 4, 16), past_key_values=None)
model(torch.randn(1, 4, 16), past_key_values=None)
assert calls == [True, True]
handle.remove()
@pytest.mark.parametrize("granularity", ["tensor", "head"])
def test_fp8_update_quantizes_and_keeps_running_max_scale(granularity):
cache = QuantizedKVParameterCache(dtype="fp8", granularity=granularity)
key_states = torch.linspace(-3.1, 2.7, 64).reshape(1, 2, 2, 16)
value_states = key_states * 2.7
actual_keys, actual_values = cache.update(key_states, value_states, 0)
assert not torch.equal(actual_keys, key_states)
assert not torch.equal(actual_values, value_states)
initial_k_scale = cache.k_scales[0].clone()
initial_v_scale = cache.v_scales[0].clone()
cache.update(key_states / 4, value_states / 4, 0)
torch.testing.assert_close(cache.k_scales[0], initial_k_scale)
torch.testing.assert_close(cache.v_scales[0], initial_v_scale)
@pytest.mark.parametrize("granularity", ["tensor", "head"])
def test_fp8_update_running_max_per_side_head_and_layer(granularity):
cache = QuantizedKVParameterCache(dtype="fp8", granularity=granularity)
first_keys = torch.tensor([8.0, 1.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)
first_values = torch.tensor([2.0, 16.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)
cache.update(first_keys, first_values, 2)
second_keys = torch.tensor([2.0, 4.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)
second_values = torch.tensor([32.0, 1.0]).reshape(1, 2, 1, 1).expand(1, 2, 2, 16)
cache.reset_states()
cache.update(second_keys, second_values, 2)
expected_keys = torch.tensor([8.0, 4.0] if granularity == "head" else [8.0]) / 448
expected_values = torch.tensor([32.0, 16.0] if granularity == "head" else [32.0]) / 448
torch.testing.assert_close(cache.k_scales[2], expected_keys)
torch.testing.assert_close(cache.v_scales[2], expected_values)
assert not cache.k_scales[2].requires_grad
assert not cache.v_scales[2].requires_grad
cache.update(second_keys / 8, second_values / 8, 0)
expected_layer_keys = torch.tensor([0.25, 0.5] if granularity == "head" else [0.5]) / 448
expected_layer_values = torch.tensor([4.0, 0.125] if granularity == "head" else [4.0]) / 448
torch.testing.assert_close(cache.k_scales[0], expected_layer_keys)
torch.testing.assert_close(cache.v_scales[0], expected_layer_values)
torch.testing.assert_close(cache.k_scales[2], expected_keys)
torch.testing.assert_close(cache.v_scales[2], expected_values)
@pytest.mark.parametrize("scheme", ["W4A16", "NVFP4"])
def test_nvfp4_observer_preserves_autoround_tuned_weights(scheme):
import copy
from contextlib import nullcontext
from tokenizers import Tokenizer
from tokenizers.models import WordLevel
from transformers import LlamaConfig, LlamaForCausalLM, PreTrainedTokenizerFast
from auto_round import AutoRound
from auto_round.experimental.kv_cache import kvcache_quant_context
torch.manual_seed(42)
config = LlamaConfig(
vocab_size=16,
hidden_size=32,
intermediate_size=32,
num_hidden_layers=1,
num_attention_heads=2,
num_key_value_heads=2,
max_position_embeddings=16,
attn_implementation="eager",
)
baseline = LlamaForCausalLM(config).eval()
observed = copy.deepcopy(baseline)
original_weight = baseline.model.layers[0].self_attn.k_proj.weight.detach().clone()
tokenizer = PreTrainedTokenizerFast(
tokenizer_object=Tokenizer(WordLevel({str(index): index for index in range(config.vocab_size)}, unk_token="0")),
unk_token="0",
pad_token="0",
eos_token="1",
)
dataset = torch.randint(0, config.vocab_size, (2, 4))
for model, dtype in ((baseline, None), (observed, "nvfp4")):
compressor = AutoRound(
model,
tokenizer=tokenizer,
scheme=scheme,
group_size=16,
iters=2,
batch_size=1,
nsamples=2,
seqlen=4,
dataset=[{"input_ids": sample.unsqueeze(0).clone()} for sample in dataset],
device_map="cpu",
amp=False,
seed=42,
static_kv_dtype=dtype,
)
context = nullcontext() if dtype is None else kvcache_quant_context(model, dtype)
with context:
compressor.quantize()
for name, expected in baseline.named_parameters():
torch.testing.assert_close(observed.get_parameter(name), expected)
assert not torch.equal(baseline.model.layers[0].self_attn.k_proj.weight, original_weight)
for layer in observed.model.layers:
assert layer.self_attn.k_global_scale.item() > 0
assert layer.self_attn.v_global_scale.item() > 0
with torch.no_grad():
torch.testing.assert_close(observed(dataset).logits, baseline(dataset).logits)