PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
import os
import pytest
import torch
from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
from llmcompressor import oneshot
from llmcompressor.modifiers.autoround import AutoRoundModifier
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound, AWQConfig
from auto_round.calib_dataset import get_dataset
from ...unit.envs import multi_card
recipe_str = """
quant_stage:
quant_modifiers:
AutoRoundModifier:
ignore: ["lm_head"]
iters: 10
config_groups:
group_0:
targets:
- "Linear"
input_activations: null
output_activations: null
weights:
num_bits: 4
type: "int"
symmetric: true
strategy: group
group_size: 128
"""
recipe_modifier_full = AutoRoundModifier(
ignore=["lm_head"],
iters=10,
config_groups={
"group_0": QuantizationScheme(
targets=["Linear"],
weights=QuantizationArgs(num_bits=4, strategy="group", group_size=128),
)
},
)
recipe_modifier_nvfp4 = AutoRoundModifier(
ignore=["lm_head"],
iters=2,
scheme="NVFP4",
)
recipe_modifier_mxfp4 = AutoRoundModifier(
ignore=["lm_head"],
iters=0,
scheme="MXFP4",
)
w8a8_dynamic_recipe_modifier = AutoRoundModifier(
ignore=["lm_head"],
iters=0,
enable_torch_compile=False,
config_groups={
"group_0": QuantizationScheme(
targets=["Linear"],
weights=QuantizationArgs(num_bits=8, type="float", strategy="channel"),
input_activations=QuantizationArgs(num_bits=8, type="float", strategy="token", dynamic=True),
)
},
)
w8a8_static_recipe_modifier = AutoRoundModifier(
ignore=["lm_head"],
iters=0,
enable_torch_compile=False,
config_groups={
"group_0": QuantizationScheme(
targets=["Linear"],
weights=QuantizationArgs(num_bits=8, type="float", strategy="tensor"),
input_activations=QuantizationArgs(num_bits=8, type="float", strategy="tensor"),
)
},
)
@pytest.mark.parametrize(
"recipe",
[
recipe_str,
recipe_modifier_full,
recipe_modifier_nvfp4,
recipe_modifier_mxfp4,
],
)
def test_oneshot_application(tiny_tiny_llama_model_path, recipe, tmp_path):
output = tmp_path / "oneshot_output"
model = tiny_tiny_llama_model_path
tokenizer = AutoTokenizer.from_pretrained(model)
dataset = get_dataset(
tokenizer=tokenizer,
seqlen=1024,
nsamples=32,
)
device = "cuda:0" if torch.cuda.is_available() else "cpu"
oneshot(
model=model,
dataset=dataset,
output_dir=output,
recipe=recipe,
)
model_loaded = AutoModelForCausalLM.from_pretrained(output, device_map=device)
# Check that the model is quantized
# decompress() will attach a quantization_config to the model
# as we decompress right away
quantization_config = model_loaded.config.quantization_config.quantization_config
assert quantization_config is not None
# check config is set properly
assert "lm_head" in quantization_config.ignore
assert len(quantization_config.config_groups) == 1
quant_scheme = quantization_config.config_groups["group_0"]
assert isinstance(quant_scheme, QuantizationScheme)
weight_args = quantization_config.config_groups["group_0"].weights
assert isinstance(weight_args, QuantizationArgs)
assert weight_args.num_bits == 4
# Check a specific layer is quantized
targeted_linear_layer = model_loaded.model.layers[2].self_attn.q_proj
assert hasattr(targeted_linear_layer, "quantization_scheme")
# Check lm-head is not quantized
not_targeted = model_loaded.lm_head
assert not hasattr(not_targeted, "quantization_scheme")
@multi_card
def test_oneshot_with_device_ids(tiny_tiny_llama_model_path, tmp_path):
output = tmp_path / "oneshot_output"
model = tiny_tiny_llama_model_path
tokenizer = AutoTokenizer.from_pretrained(model)
dataset = get_dataset(
tokenizer=tokenizer,
seqlen=512,
nsamples=4,
)
device = "cuda:0" if torch.cuda.is_available() else "cpu"
recipe = AutoRoundModifier(
ignore=["lm_head"],
iters=10,
config_groups={
"group_0": QuantizationScheme(
targets=["Linear"],
weights=QuantizationArgs(num_bits=4, strategy="group", group_size=128),
)
},
device_ids="0,1",
)
oneshot(
model=model,
dataset=dataset,
output_dir=output,
recipe=recipe,
)
model_loaded = AutoModelForCausalLM.from_pretrained(output, device_map=device)
# Check that the model is quantized
# decompress() will attach a quantization_config to the model
# as we decompress right away
quantization_config = model_loaded.config.quantization_config.quantization_config
assert quantization_config is not None
# check config is set properly
assert "lm_head" in quantization_config.ignore
assert len(quantization_config.config_groups) == 1
quant_scheme = quantization_config.config_groups["group_0"]
assert isinstance(quant_scheme, QuantizationScheme)
weight_args = quantization_config.config_groups["group_0"].weights
assert isinstance(weight_args, QuantizationArgs)
assert weight_args.num_bits == 4
# Check a specific layer is quantized
targeted_linear_layer = model_loaded.model.layers[2].self_attn.q_proj
assert hasattr(targeted_linear_layer, "quantization_scheme")
# Check lm-head is not quantized
not_targeted = model_loaded.lm_head
assert not hasattr(not_targeted, "quantization_scheme")
@pytest.mark.parametrize(
"recipe",
[w8a8_dynamic_recipe_modifier, w8a8_static_recipe_modifier],
)
def test_rtn_oneshot(recipe, tmp_path, tiny_tiny_llama_model_path):
output = tmp_path / "oneshot_output"
model = tiny_tiny_llama_model_path
tokenizer = AutoTokenizer.from_pretrained(model)
dataset = get_dataset(
tokenizer=tokenizer,
seqlen=1024,
nsamples=32,
)
device = "cuda:0" if torch.cuda.is_available() else "cpu"
oneshot(
model=model,
dataset=dataset,
output_dir=output,
recipe=recipe,
)
model_loaded = AutoModelForCausalLM.from_pretrained(output, device_map=device)
quantization_config = model_loaded.config.quantization_config.quantization_config
assert quantization_config is not None
# check config is set properly
assert "lm_head" in quantization_config.ignore
assert len(quantization_config.config_groups) == 1
quant_scheme = quantization_config.config_groups["group_0"]
assert isinstance(quant_scheme, QuantizationScheme)
weight_args = quantization_config.config_groups["group_0"].weights
act_args = quantization_config.config_groups["group_0"].input_activations
assert isinstance(weight_args, QuantizationArgs)
assert weight_args.num_bits == recipe.config_groups["group_0"].weights.num_bits
assert weight_args.strategy == recipe.config_groups["group_0"].weights.strategy
if act_args is not None:
assert act_args.num_bits == recipe.config_groups["group_0"].input_activations.num_bits
assert act_args.strategy == recipe.config_groups["group_0"].input_activations.strategy
# Check a specific layer is quantized
targeted_linear_layer = model_loaded.model.layers[2].self_attn.q_proj
assert hasattr(targeted_linear_layer, "quantization_scheme")
# Check lm-head is not quantized
not_targeted = model_loaded.lm_head
assert not hasattr(not_targeted, "quantization_scheme")
# ---------------------------------------------------------------------------
# AWQ W8A8 llm_compressor export config args
# ---------------------------------------------------------------------------
def test_llmc_awq_w8a8_export_config_args(tiny_opt_model_path, tmp_path):
"""W8A8 AWQ → llm_compressor: verify compressed-tensors metadata fields."""
save_dir = str(tmp_path / "saved")
ar = AutoRound(
tiny_opt_model_path,
scheme="INT8",
alg_configs=AWQConfig(),
nsamples=2,
seqlen=32,
batch_size=2,
)
_, save_path = ar.quantize_and_save(output_dir=save_dir, format="llm_compressor")
config = AutoConfig.from_pretrained(save_path, trust_remote_code=True)
qconfig = config.quantization_config
assert qconfig["quant_method"] == "compressed-tensors"
group0 = qconfig["config_groups"]["group_0"]
# Weight args
assert group0["weights"]["num_bits"] == 8
assert group0["weights"]["type"] == "int"
assert group0["weights"]["symmetric"] is True
# Activation args
assert group0["input_activations"]["num_bits"] == 8
# Targets
targets = group0.get("targets")
assert targets is not None and len(targets) > 0
# QuantLinear check: verify saved weights are int8 with per-channel scales
from safetensors import safe_open
st_files = [f for f in os.listdir(save_path) if f.endswith(".safetensors")]
assert len(st_files) > 0, f"No safetensors files in {save_path}"
with safe_open(os.path.join(save_path, st_files[0]), framework="pt") as f:
weight = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight")
assert weight.dtype == torch.int8, f"Expected int8 weight, got {weight.dtype}"
scale = f.get_tensor("model.decoder.layers.0.self_attn.k_proj.weight_scale")
assert scale.shape[1] == 1, f"Expected per-channel scale shape (out, 1), got {scale.shape}"