PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
import shutil
from test.helpers import get_model_path, lamini_name_or_path
import pytest
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
class TestQuantizationBlocks:
@classmethod
def setup_class(self):
self.model_name = lamini_name_or_path
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
@classmethod
def teardown_class(self):
shutil.rmtree("runs", ignore_errors=True)
@pytest.fixture(autouse=True)
def setup_save_dir(self, tmp_path):
self.save_dir = str(tmp_path / "saved")
def test_block_name_quant(self, dataloader):
self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
from auto_round.utils import get_block_names
llm_block_names = get_block_names(self.model)
bits, group_size, sym, batch_size = 4, 128, True, 20
autoround = AutoRound(
self.model,
self.tokenizer,
bits=bits,
group_size=group_size,
sym=sym,
iters=2,
seqlen=2,
batch_size=batch_size,
dataset=dataloader,
to_quant_block_names=llm_block_names,
)
autoround.quantize()
quantized_model_path = self.save_dir
autoround.save_quantized(quantized_model_path, inplace=False, safe_serialization=False, format="auto_round")
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
quant_config = model.config.quantization_config
assert quant_config.block_name_to_quantize is not None
def test_mm_block_name(self, tiny_qwen_vl_model_path):
from transformers import Qwen2VLForConditionalGeneration
from auto_round.utils import get_block_names
model_name = tiny_qwen_vl_model_path
model = Qwen2VLForConditionalGeneration.from_pretrained(model_name, trust_remote_code=True, device_map="auto")
block_name = get_block_names(model, quant_vision=True)
assert len(block_name) == 2
assert all("visual.merger.mlp" not in n for n in block_name)
block_name = get_block_names(model, quant_vision=False)
assert len(block_name) == 1
assert block_name == get_block_names(model)
def test_moe(self, tiny_qwen_moe_model_path):
from transformers import AutoConfig
from auto_round.utils import get_block_names
config = AutoConfig.from_pretrained(tiny_qwen_moe_model_path, trust_remote_code=True)
model = AutoModelForCausalLM.from_config(config)
block_name = get_block_names(model)
block_name_2 = get_block_names(model, quant_vision=True)
assert block_name == block_name_2
assert len(block_name_2) == 1
assert block_name_2[0][-1] == "model.layers.1"