PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
import copy
import shutil
import tempfile
from test.helpers import get_model_path, save_tiny_model
import pytest
import torch
import transformers
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
from auto_round import schemes as ar_schemes
from auto_round.experimental import qmodules as ar_qmodules
from auto_round.export.export_to_autoround import qlinear_fp as ar_qlinear_fp
from auto_round.export.formats import BackendDataType
from ...envs import has_module, require_awq, require_optimum
testing_schemes = [
BackendDataType.MXFP8.value,
BackendDataType.MXFP4.value,
BackendDataType.NVFP4.value,
]
QMODULE_MAPPING = {
BackendDataType.MXFP8.value: ar_qmodules.MXFP8QuantLinear,
BackendDataType.MXFP4.value: ar_qmodules.MXFP4QuantLinear,
BackendDataType.NVFP4.value: ar_qmodules.NVFP4QuantLinear,
}
@pytest.mark.parametrize("scheme", testing_schemes)
@torch.inference_mode()
def test_e2e_quant_and_infer(scheme, tiny_qwen_model_path):
# Use a temporary directory for saving the quantized model
with tempfile.TemporaryDirectory() as temp_dir:
# Load the tokenizer and model
tokenizer = AutoTokenizer.from_pretrained(tiny_qwen_model_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(
tiny_qwen_model_path,
device_map="cpu",
torch_dtype="auto",
trust_remote_code=True,
)
# Initialize AutoRound for quantization
autoround = AutoRound(
model,
tokenizer,
scheme=scheme,
iters=0,
nsamples=2,
disable_opt_rtn=True,
)
# Quantize and save the model to the temporary directory
quantized_model_path = f"{temp_dir}/tmp_autoround_{scheme}"
_, quantized_model_path = autoround.quantize_and_save(format="auto_round", output_dir=quantized_model_path)
# Perform inference with the quantized model
model = AutoModelForCausalLM.from_pretrained(
quantized_model_path,
torch_dtype="auto",
)
model.eval()
assert has_module(model, QMODULE_MAPPING[scheme]), f"Expected {QMODULE_MAPPING[scheme].__name__} in the model."
# Skip accuracy check for tiny model.
# tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
# prompt = "The capital of France is"
# encode = tokenizer.encode(prompt, return_tensors="pt")
# output_tokens = model.generate(
# encode,
# max_length=10,
# )
# output = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
# assert "paris" in output.lower(), f"Expected 'Paris' in the output, but got: {output}"
class TestAutoRound:
@pytest.fixture(autouse=True)
def _save_dir(self, tmp_path):
self.save_dir = str(tmp_path / "saved")
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
@pytest.fixture(autouse=True, scope="class")
def setup_and_teardown_class(self):
yield
shutil.rmtree("runs", ignore_errors=True)
def test_nvfp4_moe_actmax_rtn(self, tiny_deepseek_v2_model_path, dataloader):
# model_name = "/data0/deepseek-ai/DeepSeek-V2-Lite"
scheme = "nvfp4"
autoround = AutoRound(
tiny_deepseek_v2_model_path,
scheme=scheme,
iters=0,
seqlen=2,
nsamples=2,
dataset=dataloader,
trust_remote_code=False,
)
autoround.quantize()
quantized_model_path = self.save_dir
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map="auto")
print(model)
assert model is not None, "Failed to load the quantized model."
@pytest.mark.skip_ci(reason="Matrix: Cannot test all cases in CI; time-consuming")
def test_nvfp4_moe_actmax_ar(self, tiny_deepseek_v2_model_path, dataloader):
scheme = "nvfp4"
autoround = AutoRound(
tiny_deepseek_v2_model_path,
scheme=scheme,
iters=1,
seqlen=2,
nsamples=2,
dataset=dataloader,
trust_remote_code=False,
)
autoround.quantize()
quantized_model_path = self.save_dir
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")