PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
import json
import os
import shutil
from test.helpers import check_version, forbid_threaded_packing, get_model_path, opt_name_or_path
import pytest
import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound, AutoScheme
from auto_round.export.export_to_llmcompressor import export_to_fp as llmc_fp_export
from auto_round.export.export_to_llmcompressor import export_to_static_fp as llmc_static_fp_export
from ...envs import is_compressed_tensors_available
pytestmark = pytest.mark.skipif(not is_compressed_tensors_available(), reason="test requires compressed-tensors")
@pytest.mark.parametrize("bits", [2, 3, 4, 5, 6, 7, 8])
def test_llmc_fp_export_scheme_supports_weight_only_integer_widths(bits):
from auto_round.export.export_to_llmcompressor.export_to_fp import _get_scheme
assert _get_scheme(bits, "int", act_bits=16) == f"W{bits}A16"
def test_llmc_fp_export_scheme_requires_weight_only_activations():
from auto_round.export.export_to_llmcompressor.export_to_fp import _get_scheme
assert _get_scheme(4, "int", act_bits=8) is None
assert _get_scheme(9, "int", act_bits=16) is None
@pytest.mark.parametrize("data_type", ["fp", "float"])
def test_llmc_format_accepts_full_precision_dtype_aliases(data_type, monkeypatch):
from types import SimpleNamespace
from auto_round.export.formats.backends.llm_compressor import LLMCompressorFormat
from auto_round.schemes import QuantizationScheme
monkeypatch.setattr(
"auto_round.export.export_to_llmcompressor.check_compressed_tensors_supported",
lambda **kwargs: None,
)
scheme = QuantizationScheme(bits=16, data_type=data_type, act_bits=16, act_data_type="float")
output_format = LLMCompressorFormat("llm_compressor", scheme, SimpleNamespace(mllm=False))
assert output_format.backend.output_format == "llm_compressor:mx_fp"
class TestLLMCZpInt8:
"""pack_layer must hand compressed_tensors an int8 zero-point tensor:
NeUQI stores zp as a float32 integral-valued tensor, and CT's packer
(pack_to_int32) hard-requires torch.int8."""
def _make_layer(self, zp_tensor, bits=4, sym=False):
import torch.nn as nn
layer = nn.Linear(16, 8)
layer.bits = bits
layer.sym = sym
layer.group_size = 16
layer.data_type = "int"
layer.act_bits = 16
layer.act_sym = True
layer.act_data_type = None
layer.scale = torch.full((8, 1), 0.01)
layer.zp = zp_tensor
return layer
def test_tensor_zp_cast_to_int8(self):
import torch
from auto_round.export.export_to_llmcompressor.export import pack_layer
layer = self._make_layer(torch.full((8, 1), 5.0)) # float32 integral-valued
model = torch.nn.Sequential()
model.add_module("q_proj", layer)
pack_layer("q_proj", model, device="cpu")
# after compression CT owns the params: zp arrives packed to int32 and
# the layer is marked COMPRESSED (the pre-fix code raised in pack_to_int32)
zp = getattr(layer, "weight_zero_point", None)
assert zp is not None and zp.dtype == torch.int32
assert getattr(layer, "weight_packed", None) is not None
def test_out_of_range_zp_raises(self):
import torch
from auto_round.export.export_to_llmcompressor.export import pack_layer
layer = self._make_layer(torch.full((8, 1), 200.0), bits=4) # zp >> 2^b-1: invalid at any convention
model = torch.nn.Sequential()
model.add_module("q_proj", layer)
try:
pack_layer("q_proj", model, device="cpu")
except ValueError as e:
assert "zero-point" in str(e)
else:
raise AssertionError("out-of-range zp must raise")
@pytest.mark.skipif(
check_version("compressed-tensors==0.20.0"),
reason="compressed-tensors 0.20.0 has a zero-point unpacking bug",
)
def test_asym_zp_signed_convention_round_trip(self):
"""AutoRound zp is unsigned (q in [0, 2^b-1], W = (q - zp) * s);
compressed-tensors re-quantizes with the SIGNED range
[-2^(b-1), 2^(b-1)-1] (clamp AFTER adding zp). The unsigned zp must be
shifted into the signed convention before packing, or CT's clamp
collapses every level above 2^(b-1)-1. Round-trip through CT's own
compressor must reproduce AutoRound's fake-quantized weight exactly."""
import torch
import torch.nn as nn
from compressed_tensors.compressors.pack_quantized import PackedQuantizationCompressor
from auto_round.export.export_to_llmcompressor.export import pack_layer
torch.manual_seed(0)
out_f, in_f, gs = 8, 32, 16
w = torch.randn(out_f, in_f)
scale = torch.rand(out_f, in_f // gs) * 0.02 + 0.005 # [out, in/g]
zp = torch.randint(0, 16, (out_f, in_f // gs)).float()
zp[:, 0] = 12 # exercise levels above 2^(b-1)-1 (the pre-fix clamp)
layer = nn.Linear(in_f, out_f)
with torch.no_grad():
layer.weight.copy_(w)
layer.bits, layer.sym, layer.group_size = 4, False, gs
layer.data_type, layer.act_bits, layer.act_sym, layer.act_data_type = "int", 16, True, None
layer.scale, layer.zp = scale, zp
model = nn.Sequential()
model.add_module("q_proj", layer)
pack_layer("q_proj", model, device="cpu")
assert layer.weight_zero_point.stride(-2) == 1
# AutoRound's own fake-quantized weight (unsigned convention)
s = scale.repeat_interleave(gs, dim=1)
z = zp.repeat_interleave(gs, dim=1)
q = (w / s + z).round().clamp(0, 15)
w_ref = (q - z) * s
state = {
"weight_packed": layer.weight_packed.detach(),
"weight_scale": layer.weight_scale.detach(),
"weight_zero_point": layer.weight_zero_point.detach(),
"weight_shape": layer.weight_shape.detach(),
}
out = PackedQuantizationCompressor.decompress(state, layer.quantization_scheme)
assert torch.equal(out["weight"].to(torch.float32), w_ref)
def test_w8_asym_pack_round_trip(self):
"""8-bit asym is the boundary case: zp spans [0, 255] and only fits int8
via the signed-convention shift. The llm_compressor format accepts W8
asym (vLLM serves it through the compressed-tensors kernel path), so
the pack must round-trip."""
import torch
import torch.nn as nn
from compressed_tensors.compressors.pack_quantized import PackedQuantizationCompressor
from auto_round.export.export_to_llmcompressor.export import pack_layer
torch.manual_seed(0)
out_f, in_f, gs = 8, 32, 16
w = torch.randn(out_f, in_f)
scale = torch.rand(out_f, in_f // gs) * 0.02 + 0.005
zp = torch.randint(0, 256, (out_f, in_f // gs)).float() # full 8-bit span
zp[:, 0] = 250 # far above 127: unsigned convention, must survive the shift
layer = nn.Linear(in_f, out_f)
with torch.no_grad():
layer.weight.copy_(w)
layer.bits, layer.sym, layer.group_size = 8, False, gs
layer.data_type, layer.act_bits, layer.act_sym, layer.act_data_type = "int", 16, True, None
layer.scale, layer.zp = scale, zp
model = nn.Sequential()
model.add_module("q_proj", layer)
pack_layer("q_proj", model, device="cpu")
s = scale.repeat_interleave(gs, dim=1)
z = zp.repeat_interleave(gs, dim=1)
q = (w / s + z).round().clamp(0, 255)
w_ref = (q - z) * s
state = {
"weight_packed": layer.weight_packed.detach(),
"weight_scale": layer.weight_scale.detach(),
"weight_zero_point": layer.weight_zero_point.detach(),
"weight_shape": layer.weight_shape.detach(),
}
out = PackedQuantizationCompressor.decompress(state, layer.quantization_scheme)
assert torch.equal(out["weight"].to(torch.float32), w_ref)
class TestLLMC:
@classmethod
def setup_class(self):
self.model_name = get_model_path("stas/tiny-random-llama-2")
self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
@classmethod
def teardown_class(self):
shutil.rmtree("runs", ignore_errors=True)
def test_mixed_int_bits_llmcompressor_format(self, tiny_opt_model_path, tmp_path):
"""Mixed int W5A16-W8A16 auto-scheme export with an unsupported-as-first-option scheme
leading the options (W5A16) must resolve and export through the llm_compressor backend
(compressed-tensors pack-quantized handles any int width in 2..8)."""
scheme = AutoScheme(
avg_bits=6.0,
options=("W5A16", "W6A16", "W7A16", "W8A16"),
)
ar = AutoRound(
model=tiny_opt_model_path,
iters=0,
disable_opt_rtn=True,
scheme=scheme,
)
_, tmp_path = ar.quantize_and_save(output_dir=tmp_path, format="auto_round:llm_compressor")
model = AutoModelForCausalLM.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)
quantization_config = model.config.quantization_config.to_dict()
assert quantization_config["format"] == "pack-quantized"
assert quantization_config["config_groups"], "no quantized config groups were exported"
for group in quantization_config["config_groups"].values():
weights = group["weights"]
assert weights["num_bits"] in (5, 6, 7, 8), f"unexpected bit-width {weights['num_bits']}"
assert weights["type"] == "int"
assert weights["symmetric"] is True
assert group["input_activations"] is None # weight-only quantization
# remove since w8a8 not in llmcompressor format supported schemes
# def test_llmcompressor_w8a8(self):
# bits, group_size, sym, act_bits = 8, -1, True, 8
# ## quantize the model
# autoround = AutoRound(
# self.model,
# self.tokenizer,
# bits=bits,
# group_size=group_size,
# sym=sym,
# act_bits=act_bits,
# seqlen=8,
# nsamples=2,
# iters=0,
# )
# autoround.quantize()
# autoround.save_quantized("./saved", format="llm_compressor", inplace=True)
def test_llmcompressor_fp8(self, tmp_path):
## quantize the model
model_name = opt_name_or_path
autoround = AutoRound(
model_name,
scheme="FP8_STATIC",
seqlen=8,
nsamples=2,
iters=0,
)
_, quantized_model_path = autoround.quantize_and_save(tmp_path, format="llm_compressor")
# from vllm import LLM
# model = LLM(tmp_path)
# result = model.generate("Hello my name is")
# print(result)
import json
from safetensors import safe_open
with open(os.path.join(quantized_model_path, "config.json")) as f:
config = json.load(f)
assert "group_0" in config["quantization_config"]["config_groups"]
assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8
assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "channel"
assert config["quantization_config"]["quant_method"] == "compressed-tensors"
f = safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt")
assert len(f.get_tensor("model.decoder.layers.0.fc1.weight_scale").shape) == 2
def test_autoround_llmcompressor_fp8(self, tmp_path):
## quantize the model
model_name = opt_name_or_path
autoround = AutoRound(
model_name,
scheme="FP8_STATIC",
seqlen=8,
group_size=0,
nsamples=2,
iters=0,
)
_, quantized_model_path = autoround.quantize_and_save(tmp_path, format="auto_round:llm_compressor")
import json
with open(os.path.join(quantized_model_path, "config.json")) as f:
config = json.load(f)
assert "group_0" in config["quantization_config"]["config_groups"]
assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8
assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "tensor"
assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["strategy"] == "tensor"
assert config["quantization_config"]["quant_method"] == "compressed-tensors"
def test_bf16_default_mxfp4_override_standard_flow(self, tiny_opt_model_path, tmp_path):
target = "model.decoder.layers.0.self_attn.q_proj"
ar = AutoRound(
model=tiny_opt_model_path,
scheme="BF16",
layer_config={target: "MXFP4"},
iters=0,
disable_opt_rtn=True,
disable_model_free=True,
)
compressed_model, output_dir = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
assert hasattr(compressed_model.model.decoder.layers[0].self_attn.q_proj, "weight_scale")
with open(os.path.join(output_dir, "config.json")) as config_file:
config = json.load(config_file)["quantization_config"]
assert config["format"] == "mxfp4-pack-quantized"
assert config["config_groups"]["group_0"]["targets"] == [target]
def test_bf16_default_w4a16_override_standard_flow(self, tiny_opt_model_path, tmp_path):
target = "model.decoder.layers.0.self_attn.q_proj"
ar = AutoRound(
model=tiny_opt_model_path,
scheme="BF16",
layer_config={target: "W4A16"},
iters=0,
disable_opt_rtn=True,
disable_model_free=True,
)
_, output_dir = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
with open(os.path.join(output_dir, "config.json")) as config_file:
config = json.load(config_file)["quantization_config"]
assert config["format"] == "pack-quantized"
assert config["config_groups"]["group_0"]["targets"] == [target]
assert config["config_groups"]["group_0"]["weights"]["num_bits"] == 4
def test_mxfp8_llmcompressor_format(self, tiny_opt_model_path, tmp_path):
scheme = "mxfp8"
ar = AutoRound(
model=tiny_opt_model_path,
iters=0,
disable_opt_rtn=True,
scheme=scheme,
disable_model_free=True,
)
compressed_model, tmp_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_proj
assert (
hasattr(tmp_layer, "weight_scale")
and hasattr(tmp_layer, "weight")
and tmp_layer.weight_scale.dtype is torch.uint8
and tmp_layer.weight_scale.shape[0] == 768
), "Illegal MXFP8 packing name or data_type or shape"
quantization_config = transformers.AutoConfig.from_pretrained(
tmp_path, trust_remote_code=True
).quantization_config
assert (
quantization_config["format"] == "mxfp8-quantized"
and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8
and quantization_config["config_groups"]["group_0"]["weights"]["group_size"] == 32
and quantization_config["config_groups"]["group_0"]["weights"]["scale_dtype"] == "torch.uint8"
and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8
and quantization_config["config_groups"]["group_0"]["input_activations"]["group_size"] == 32
and quantization_config["config_groups"]["group_0"]["input_activations"]["scale_dtype"] == "torch.uint8"
and quantization_config["ignore"] == ["lm_head"]
), f"Invalid MXFP8 quantization configuration: {quantization_config}"
def test_mxfp8_llmcompressor_kv_config(self, tiny_opt_model_path, tmp_path):
ar = AutoRound(
model=tiny_opt_model_path,
iters=0,
disable_opt_rtn=True,
scheme="mxfp8",
static_kv_dtype="fp8",
)
_, quantized_model_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
with open(os.path.join(quantized_model_path, "config.json")) as f:
config = json.load(f)
kv_cache_scheme = config["quantization_config"]["kv_cache_scheme"]
assert kv_cache_scheme is not None
assert kv_cache_scheme["num_bits"] == 8
assert kv_cache_scheme["type"] == "float"
assert kv_cache_scheme["strategy"] == "tensor"
assert kv_cache_scheme["dynamic"] is False
assert kv_cache_scheme["symmetric"] is True
def test_mxfp8_llmcompressor_per_head_kv_config(self, tiny_opt_model_path, tmp_path):
from safetensors import safe_open
ar = AutoRound(
model=tiny_opt_model_path,
iters=0,
disable_opt_rtn=True,
scheme="mxfp8",
static_kv_dtype="fp8",
static_kv_granularity="head",
)
compressed_model, quantized_model_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
with open(os.path.join(quantized_model_path, "config.json")) as f:
config = json.load(f)
kv_cache_scheme = config["quantization_config"]["kv_cache_scheme"]
assert kv_cache_scheme is not None
assert kv_cache_scheme["strategy"] == "attn_head"
num_kv_heads = compressed_model.config.num_attention_heads
if hasattr(compressed_model.config, "num_key_value_heads"):
num_kv_heads = compressed_model.config.num_key_value_heads
with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:
k_scale = f.get_tensor("model.decoder.layers.0.self_attn.k_scale")
v_scale = f.get_tensor("model.decoder.layers.0.self_attn.v_scale")
assert k_scale.shape == torch.Size([num_kv_heads])
assert v_scale.shape == torch.Size([num_kv_heads])
def test_mxfp8_llmcompressor_attention_config(self, tiny_opt_model_path, tmp_path):
from safetensors import safe_open
ar = AutoRound(
model=tiny_opt_model_path,
iters=0,
disable_opt_rtn=True,
scheme="mxfp8",
static_attention_dtype="fp8",
)
compressed_model, quantized_model_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
with open(os.path.join(quantized_model_path, "config.json")) as f:
saved_config = json.load(f)
# Attention config is stored in a dedicated top-level field (not config_groups).
attention_config = saved_config["quantization_config"]["attention_input_activations"]
assert attention_config is not None
assert attention_config["targets"] == [compressed_model.model.decoder.layers[0].self_attn.__class__.__name__]
assert attention_config["input_activations"]["num_bits"] == 8
assert attention_config["input_activations"]["type"] == "float"
assert attention_config["input_activations"]["strategy"] == "tensor"
assert attention_config["input_activations"]["dynamic"] is False
assert attention_config["input_activations"]["symmetric"] is True
assert saved_config["quantization_config"]["kv_cache_scheme"] is not None
quantization_config = transformers.AutoConfig.from_pretrained(
quantized_model_path, trust_remote_code=True
).quantization_config
config_groups = quantization_config["config_groups"]
# Only Linear layers should be in config_groups.
for group_targets in (g["targets"] for g in config_groups.values()):
assert "Linear" in group_targets, f"Unexpected non-Linear targets in config_groups: {group_targets}"
assert quantization_config["kv_cache_scheme"] is not None
assert getattr(compressed_model.model.decoder.layers[0].self_attn, "q_scale", None) is not None
with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as checkpoint:
keys = list(checkpoint.keys())
assert any(key.endswith(".self_attn.q_scale") for key in keys), "q_scale not found in checkpoint"
assert not any(key.endswith(".q_max") for key in keys), "q_max should not be exported"
def test_mxfp8_llmcompressor_per_head_attention_config(self, tiny_opt_model_path, tmp_path):
from safetensors import safe_open
ar = AutoRound(
model=tiny_opt_model_path,
iters=0,
disable_opt_rtn=True,
scheme="mxfp8",
static_attention_dtype="fp8",
static_attention_granularity="head",
)
compressed_model, quantized_model_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
with open(os.path.join(quantized_model_path, "config.json")) as f:
saved_config = json.load(f)
attention_config = saved_config["quantization_config"]["attention_input_activations"]
assert attention_config is not None
assert attention_config["targets"] == [compressed_model.model.decoder.layers[0].self_attn.__class__.__name__]
assert attention_config["input_activations"]["strategy"] == "attn_head"
assert saved_config["quantization_config"]["kv_cache_scheme"]["strategy"] == "attn_head"
with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:
q_scale = f.get_tensor("model.decoder.layers.0.self_attn.q_scale")
k_scale = f.get_tensor("model.decoder.layers.0.self_attn.k_scale")
assert q_scale.shape == torch.Size([compressed_model.config.num_attention_heads])
assert k_scale.ndim == 1
def test_mixed_precision_llmcompressor_format(self, tiny_opt_model_path, tmp_path):
scheme = AutoScheme(
avg_bits=7,
options=("MXFP4", "MXFP8"),
shared_layers=["q_proj", "k_proj", "v_proj"],
)
ar = AutoRound(
model=tiny_opt_model_path,
iters=0,
disable_opt_rtn=True,
scheme=scheme,
)
_, tmp_path = ar.quantize_and_save(output_dir=tmp_path, format="llm_compressor")
model = AutoModelForCausalLM.from_pretrained(tmp_path, torch_dtype="auto", trust_remote_code=True)
op = model.model.decoder.layers[0].fc1
if op.quantization_scheme.targets != ["Linear"]:
assert (
op.quantization_scheme.weights.num_bits == 8
and op.quantization_scheme.input_activations.num_bits == 8
and op.quantization_scheme.weights.group_size == 32
and op.quantization_scheme.input_activations.group_size == 32
and op.quantization_scheme.weights.scale_dtype == torch.uint8
and op.quantization_scheme.input_activations.scale_dtype == torch.uint8
), "Illegal MXFP4 packing name or data_type or shape"
quantization_config = model.config.quantization_config.to_dict()
assert (
quantization_config["format"] == "mixed-precision"
and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 8
and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8
and quantization_config["config_groups"]["group_0"]["format"] == "mxfp8-quantized"
and quantization_config["config_groups"]["group_1"]["weights"]["num_bits"] == 4
and quantization_config["config_groups"]["group_1"]["input_activations"]["num_bits"] == 4
and quantization_config["config_groups"]["group_1"]["format"] == "mxfp4-pack-quantized"
and quantization_config["ignore"] == ["lm_head"]
), f"Invalid mixed precision quantization configuration: {quantization_config}"
class TestLLMCNVFP4KV:
"""NVFP4 static KV cache: calibration, saved global scales, and CT-parity export."""
@staticmethod
def _expected_nvfp4_kv_args():
from compressed_tensors.quantization import preset_name_to_scheme
return preset_name_to_scheme("NVFP4", ["Linear"]).input_activations.model_dump()
def _quantize_nvfp4_kv(self, tiny_opt_model_path, dataloader, tmp_path):
return AutoRound(
tiny_opt_model_path,
scheme="NVFP4",
seqlen=8,
nsamples=2,
iters=0,
dataset=dataloader,
static_kv_dtype="nvfp4",
).quantize_and_save(tmp_path, format="llm_compressor")
def test_nvfp4_kv_missing_scales_raises(self):
"""An explicit NVFP4 KV request must not silently drop the scheme."""
import torch.nn as nn
class FakeAttention(nn.Module):
def __init__(self):
super().__init__()
self.layer_idx = 0
self.head_dim = 64
self.k_proj = nn.Linear(64, 256, bias=False)
self.v_proj = nn.Linear(64, 256, bias=False)
class FakeModel(nn.Module):
def __init__(self):
super().__init__()
self.attn = FakeAttention()
with pytest.raises(ValueError, match="NVFP4"):
llmc_fp_export._resolve_kv_cache_scheme(
FakeModel(),
use_fp8_kv=False,
use_nvfp4_kv=True,
use_fp8_attention=False,
)
@pytest.mark.timeout(120)
def test_llmcompressor_nvfp4_kv_config(self, tiny_opt_model_path, dataloader, tmp_path):
_, quantized_model_path = self._quantize_nvfp4_kv(tiny_opt_model_path, dataloader, tmp_path)
with open(os.path.join(quantized_model_path, "config.json")) as f:
config = json.load(f)
kv_cache_scheme = config["quantization_config"]["kv_cache_scheme"]
assert kv_cache_scheme is not None
# Must match the compressed-tensors NVFP4 preset input-activation args
# (same convention llm-compressor serializes for NVFP4 KV cache).
expected = self._expected_nvfp4_kv_args()
for key, value in expected.items():
assert (
kv_cache_scheme.get(key) == value
), f"kv_cache_scheme[{key!r}] mismatch: {kv_cache_scheme.get(key)!r} != {value!r}"
assert config["quantization_config"]["format"] == "nvfp4-pack-quantized"
@pytest.mark.timeout(120)
def test_llmcompressor_nvfp4_kv_global_scales(self, tiny_opt_model_path, dataloader, tmp_path):
from safetensors import safe_open
from auto_round.data_type.nvfp import calculate_gparam
from auto_round.experimental.kv_cache import QuantizedKVParameterCache
_, quantized_model_path = self._quantize_nvfp4_kv(tiny_opt_model_path, dataloader, tmp_path)
with open(os.path.join(quantized_model_path, "config.json")) as f:
config = json.load(f)
num_layers = config["num_hidden_layers"]
# Calibration must have observed KV magnitudes (data-driven path).
cache = QuantizedKVParameterCache._instance
assert cache is not None and cache.is_nvfp4
assert len(cache.k_amax) >= num_layers and len(cache.v_amax) >= num_layers
with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:
for i in range(num_layers):
k_global_scale = f.get_tensor(f"model.decoder.layers.{i}.self_attn.k_global_scale")
v_global_scale = f.get_tensor(f"model.decoder.layers.{i}.self_attn.v_global_scale")
assert k_global_scale.dtype == torch.float32 and k_global_scale.shape == torch.Size([1])
assert v_global_scale.dtype == torch.float32 and v_global_scale.shape == torch.Size([1])
assert cache.k_amax[i] > 0 and cache.v_amax[i] > 0
# vLLM serving convention: the checkpoint stores the dequantization
# multiplier amax / 2688 = 1 / calculate_gparam(amax) (see
# _nvfp4_global_scale), not the weight-style 2688 / amax.
expected_k = 1.0 / calculate_gparam(cache.k_amax[i])
expected_v = 1.0 / calculate_gparam(cache.v_amax[i])
assert torch.isclose(k_global_scale.flatten()[0], expected_k, rtol=1e-5, atol=0.0)
assert torch.isclose(v_global_scale.flatten()[0], expected_v, rtol=1e-5, atol=0.0)
@pytest.mark.timeout(120)
def test_nvfp4_kv_scheme_round_trip_matches_vllm_runtime(self, tiny_opt_model_path, dataloader, tmp_path):
from safetensors import safe_open
from auto_round.data_type.nvfp import cast_to_fp4, nv_fp4_with_static_gs
from auto_round.experimental.kv_cache import QuantizedKVParameterCache
# The stored scale must make the *vLLM* NVFP4 KV store/read kernels
# reproduce the calibration-time QDQ. The pure-Python compressed-tensors
# runtime (forward_quantize) is no longer a valid reference here: it
# interprets the checkpoint value as the weight-style global scale
# (sf = gs * block_max / 6), i.e. the opposite convention of vLLM
# (sf = (1 / k_scale) * block_max / 6). vLLM is the target serving
# engine, so this test mirrors the vLLM kernel math instead.
_, quantized_model_path = self._quantize_nvfp4_kv(tiny_opt_model_path, dataloader, tmp_path)
with open(os.path.join(quantized_model_path, "config.json")) as f:
config = json.load(f)
with safe_open(os.path.join(quantized_model_path, "model.safetensors"), framework="pt") as f:
k_global_scale = f.get_tensor("model.decoder.layers.0.self_attn.k_global_scale")
val = k_global_scale.to(torch.float32).flatten()[0]
torch.manual_seed(0)
k_states = torch.randn(1, 2, 16, 64)
# vLLM store kernel (reshape_and_cache_nvfp4, via cvt_warp_fp16_to_fp4
# with SFScaleVal = 1/val): sf = (1/val) * block_max/6 rounded to
# fp8_e4m3; fp4 code = x * (1/val) / sf (saturating at +/-6).
xb = k_states.to(torch.float32).unflatten(-1, (-1, 16))
block_max = xb.abs().amax(dim=-1, keepdim=True)
sf8 = ((block_max / 6.0) / val).to(torch.float8_e4m3fn).to(torch.float32)
out_scale = torch.where(sf8 == 0, torch.zeros_like(sf8), (1.0 / val) / sf8)
fp4 = cast_to_fp4((xb * out_scale).clamp(-6.0, 6.0))
# vLLM reader: x_hat = fp4 * sf * val
vllm_runtime_qdq = (fp4 * sf8 * val).flatten(-2).to(k_states.dtype)
assert not torch.allclose(vllm_runtime_qdq, k_states), "quantization had no effect"
# Calibration-time QDQ simulation (same amax that produced `val`):
# the fp8 block scales must be bit-identical and the dequantized
# values may only differ by fp32 associativity noise -- far below
# one fp4 step (amax / 3), which a wrong scale convention would
# produce (e.g. zeroed blocks under the old 2688 / amax convention).
cache = QuantizedKVParameterCache._instance
assert cache is not None and cache.is_nvfp4
sim_qdq, sim_scale, _ = nv_fp4_with_static_gs(k_states, tensor_max=cache.k_amax[0])
assert torch.equal(sf8, sim_scale.reshape(sf8.shape)), "fp8 block scales differ from calibration"
max_diff = (vllm_runtime_qdq - sim_qdq).abs().max().item()
assert max_diff <= 1e-4 * k_states.abs().max().item(), f"max diff {max_diff}"
@pytest.mark.timeout(120)
def test_nvfp4_kv_conflicts_with_fp8_static(self, tiny_opt_model_path, dataloader, tmp_path):
autoround = AutoRound(
tiny_opt_model_path,
scheme="FP8_STATIC",
seqlen=8,
nsamples=2,
iters=0,
dataset=dataloader,
static_kv_dtype="nvfp4",
)
with pytest.raises(ValueError, match="NVFP4"):
autoround.quantize_and_save(tmp_path, format="llm_compressor")
def test_llmcompressor_static_fp_export_packs_serially(tiny_opt_model_path, dataloader, tmp_path, monkeypatch):
autoround = AutoRound(
tiny_opt_model_path,
scheme="FP8_STATIC",
seqlen=8,
nsamples=2,
iters=0,
dataset=dataloader,
)
autoround.quantize()
forbid_threaded_packing(monkeypatch, llmc_static_fp_export)
autoround.save_quantized(tmp_path, format="llm_compressor")
assert os.path.exists(os.path.join(tmp_path, "config.json"))
def test_llmcompressor_static_fp8_kv_config(tiny_opt_model_path, dataloader, tmp_path):
autoround = AutoRound(
tiny_opt_model_path,
scheme="FP8_STATIC",
seqlen=8,
nsamples=2,
iters=0,
dataset=dataloader,
static_kv_dtype="fp8",
)
_, quantized_model_path = autoround.quantize_and_save(tmp_path, format="llm_compressor")
with open(os.path.join(quantized_model_path, "config.json")) as f:
config = json.load(f)
kv_cache_scheme = config["quantization_config"]["kv_cache_scheme"]
assert kv_cache_scheme is not None
assert kv_cache_scheme["num_bits"] == 8
assert kv_cache_scheme["type"] == "float"
assert kv_cache_scheme["strategy"] == "tensor"
assert kv_cache_scheme["dynamic"] is False
assert kv_cache_scheme["symmetric"] is True
def test_llmcompressor_static_fp8_attention_config(dataloader, tmp_path):
model_name = get_model_path("stas/tiny-random-llama-2")
autoround = AutoRound(
model_name,
scheme="FP8_STATIC",
seqlen=8,
nsamples=2,
iters=0,
dataset=dataloader,
static_attention_dtype="fp8",
)
_, quantized_model_path = autoround.quantize_and_save(tmp_path, format="llm_compressor")
with open(os.path.join(quantized_model_path, "config.json")) as f:
saved_config = json.load(f)
# Attention config is stored in a dedicated top-level field (not config_groups)
# to prevent compressed_tensors from trying to compress attention modules at load time.
attention_config = saved_config["quantization_config"]["attention_input_activations"]
assert attention_config is not None
assert "Linear" not in attention_config["targets"]
assert attention_config["input_activations"]["num_bits"] == 8
assert attention_config["input_activations"]["type"] == "float"
assert attention_config["input_activations"]["strategy"] == "tensor"
assert attention_config["input_activations"]["dynamic"] is False
assert attention_config["input_activations"]["symmetric"] is True
assert saved_config["quantization_config"]["kv_cache_scheme"] is not None
# Verify the model can be loaded without crashing.
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", trust_remote_code=True)
quantization_config = model.config.quantization_config
config = quantization_config.to_dict() if hasattr(quantization_config, "to_dict") else quantization_config
config_groups = config["config_groups"]
# Only Linear layers should be in config_groups; attention modules use the
# separate attention_input_activations field.
assert "group_0" in config_groups
for group_targets in (g["targets"] for g in config_groups.values()):
assert "Linear" in group_targets, f"Unexpected non-Linear targets in config_groups: {group_targets}"
assert config["kv_cache_scheme"] is not None
# Verify KV cache quant parameters (q_scale, k_scale, v_scale) were saved
# in the checkpoint. They show as UNEXPECTED on the loaded model because the
# base HuggingFace architecture does not register them as standard parameters.
import glob as _glob
safetensors_files = _glob.glob(os.path.join(quantized_model_path, "*.safetensors"))
assert len(safetensors_files) > 0, "No safetensors file found"
from safetensors import safe_open as _safe_open
has_q_scale = False
for sf in safetensors_files:
with _safe_open(sf, framework="pt") as f:
for key in f.keys():
if "self_attn.q_scale" in key:
has_q_scale = True
break
assert has_q_scale, "q_scale not found in checkpoint"
def test_llmcompressor_mxfp8_export_packs_serially(tmp_path, monkeypatch):
autoround = AutoRound(
model=opt_name_or_path,
iters=0,
disable_opt_rtn=True,
scheme="mxfp8",
)
autoround.quantize()
forbid_threaded_packing(monkeypatch, llmc_fp_export)
compressed_model = autoround.save_quantized(output_dir=tmp_path, format="llm_compressor")
tmp_layer = compressed_model.model.decoder.layers[1].self_attn.q_proj
assert hasattr(tmp_layer, "weight_scale")
class TestLLMCZpUnsignedDomain:
"""The packed-level validation must reject zero points outside the
UNSIGNED level range [0, 2^bits-1] before the signed-convention shift; a
shifted-only check would let e.g. a raw 4-bit zp of 100 pass (shifted 92
fits int8)."""
def _make_layer(self, zp, bits=4, sym=False):
import torch.nn as nn
layer = nn.Linear(16, 8)
layer.bits = bits
layer.sym = sym
layer.group_size = 16
layer.data_type = "int"
layer.act_bits = 16
layer.act_sym = True
layer.act_data_type = None
layer.scale = torch.full((8, 1), 0.01)
layer.zp = zp
return layer
def test_tensor_zp_above_unsigned_range_rejected(self):
import torch
from auto_round.export.export_to_llmcompressor.export import pack_layer
layer = self._make_layer(torch.full((8, 1), 100.0)) # raw 100 > 15
m = torch.nn.Sequential()
m.add_module("proj", layer)
with pytest.raises(ValueError, match="unsigned level range"):
pack_layer("proj", m, device="cpu")
def test_tensor_zp_negative_rejected(self):
import torch
from auto_round.export.export_to_llmcompressor.export import pack_layer
layer = self._make_layer(torch.full((8, 1), -3.0)) # shifted -11 fits int8
m = torch.nn.Sequential()
m.add_module("proj", layer)
with pytest.raises(ValueError, match="unsigned level range"):
pack_layer("proj", m, device="cpu")
def test_scalar_zp_out_of_range_rejected(self):
from auto_round.export.export_to_llmcompressor.export import pack_layer
for bad in (17.0, -1.0): # 4-bit unsigned range is [0, 15]
layer = self._make_layer(bad)
m = torch.nn.Sequential()
m.add_module("proj", layer)
with pytest.raises(ValueError, match="unsigned level range"):
pack_layer("proj", m, device="cpu")
def test_in_range_values_pass(self):
import torch
from auto_round.export.export_to_llmcompressor.export import pack_layer
layer = self._make_layer(torch.randint(0, 16, (8, 1)).float())
m = torch.nn.Sequential()
m.add_module("proj", layer)
pack_layer("proj", m, device="cpu")
# CT's compressor consumes the int8 zp and stores its bit-packed form
assert layer.weight_zero_point.dtype == torch.int32
assert layer.weight_packed is not None
class TestLLMCZpFiniteIntegral:
"""NaN passes every min/max comparison and would be cast straight to
int8; non-integral values must not be silently rounded into a level."""
def _make_layer(self, zp, bits=4):
import torch.nn as nn
layer = nn.Linear(16, 8)
layer.bits = bits
layer.sym = False
layer.group_size = 16
layer.data_type = "int"
layer.act_bits = 16
layer.act_sym = True
layer.act_data_type = None
layer.scale = torch.full((8, 1), 0.01)
layer.zp = zp
return layer
def _expect_refusal(self, zp, match):
from auto_round.export.export_to_llmcompressor.export import pack_layer
layer = self._make_layer(zp)
m = torch.nn.Sequential()
m.add_module("proj", layer)
with pytest.raises(ValueError, match=match):
pack_layer("proj", m, device="cpu")
def test_nan_tensor_zp_rejected(self):
zp = torch.zeros(8, 1)
zp[3, 0] = float("nan")
self._expect_refusal(zp, "non-finite")
def test_non_integral_tensor_zp_rejected(self):
self._expect_refusal(torch.full((8, 1), -0.4), "non-integral")
def test_nan_scalar_zp_rejected(self):
self._expect_refusal(float("nan"), "not a finite integer")
def test_non_integral_scalar_zp_rejected(self):
self._expect_refusal(7.3, "not a finite integer")