PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
import os
import shutil
from test.helpers import transformers_version
import pytest
import torch
from packaging import version
from auto_round import AutoRound
MODEL_LIST = (
pytest.param(
"Qwen/Qwen3-0.6B-FP8",
marks=pytest.mark.skipif(
version.parse("5.16.0") <= transformers_version < version.parse("5.17.0"),
reason="fails with transformers 5.16.x",
),
),
"Qwen/Qwen3-0.6B",
)
class TestAutoRound:
@pytest.fixture(autouse=True)
def _save_dir(self, tmp_path):
self.save_dir = str(tmp_path / "saved")
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
def check_nan_inf_in_tensor(self, tensor, name=""):
return torch.isnan(tensor).any() or torch.isinf(tensor).any()
@pytest.mark.parametrize("model_name", MODEL_LIST)
def test_small_model_rtn_generation(self, model_name):
ar = AutoRound(model_name, iters=0, scheme="FP8_STATIC", nsamples=16)
model, folder = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")
# all linears except lm_head should be quantized to FP8
fp8_linear_count = 0
for name, module in model.named_modules():
if "FP8QLinear" in type(module).__name__:
assert module.weight.dtype == torch.float8_e4m3fn, f"{name} is not in FP8"
assert not self.check_nan_inf_in_tensor(
module.weight.to(torch.float32)
), f"{name} has NaN or Inf in weights"
fp8_linear_count += 1
assert fp8_linear_count > 0, "No FP8 linear layer found in the quantized model"