Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

1.6 KB
Raw Blame
import os
import shutil
from test.helpers import transformers_version

import pytest
import torch
from packaging import version

from auto_round import AutoRound

MODEL_LIST = (
    pytest.param(
        "Qwen/Qwen3-0.6B-FP8",
        marks=pytest.mark.skipif(
            version.parse("5.16.0") <= transformers_version < version.parse("5.17.0"),
            reason="fails with transformers 5.16.x",
        ),
    ),
    "Qwen/Qwen3-0.6B",
)


class TestAutoRound:

    @pytest.fixture(autouse=True)
    def _save_dir(self, tmp_path):
        self.save_dir = str(tmp_path / "saved")
        yield
        shutil.rmtree(self.save_dir, ignore_errors=True)

    def check_nan_inf_in_tensor(self, tensor, name=""):
        return torch.isnan(tensor).any() or torch.isinf(tensor).any()

    @pytest.mark.parametrize("model_name", MODEL_LIST)
    def test_small_model_rtn_generation(self, model_name):
        ar = AutoRound(model_name, iters=0, scheme="FP8_STATIC", nsamples=16)
        model, folder = ar.quantize_and_save(output_dir=self.save_dir, format="llm_compressor")
        # all linears except lm_head should be quantized to FP8
        fp8_linear_count = 0
        for name, module in model.named_modules():
            if "FP8QLinear" in type(module).__name__:
                assert module.weight.dtype == torch.float8_e4m3fn, f"{name} is not in FP8"
                assert not self.check_nan_inf_in_tensor(
                    module.weight.to(torch.float32)
                ), f"{name} has NaN or Inf in weights"
                fp8_linear_count += 1
        assert fp8_linear_count > 0, "No FP8 linear layer found in the quantized model"