PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
import copy
import shutil
import pytest
import torch
from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
from auto_round import AutoRound
class TestAutoRoundFormatGeneration:
@pytest.fixture(autouse=True)
def setup_save_folder(self, tmp_path):
self.save_folder = str(tmp_path / "saved")
yield
shutil.rmtree(self.save_folder, ignore_errors=True)
@pytest.fixture(autouse=True)
def setup_model(self, tiny_opt_model_path):
self.model_name = tiny_opt_model_path
self.model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
yield
shutil.rmtree("runs", ignore_errors=True)
def test_4bits_sym(self, dataloader):
bits = 4
group_size = 128
sym = True
autoround = AutoRound(
self.model,
self.tokenizer,
bits=bits,
group_size=group_size,
sym=sym,
iters=1,
seqlen=2,
dataset=dataloader,
)
quantized_model_path = self.save_folder
_, quantized_model_path = autoround.quantize_and_save(
output_dir=quantized_model_path, format="auto_round", inplace=False
)
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "My name is "
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert "!!!" not in res
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu", torch_dtype=torch.float16)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert "!!!" not in res
def test_autoround_sym(self, dataloader):
for bits in [4]:
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
group_size, sym = 128, True
autoround = AutoRound(
model,
tokenizer,
bits=bits,
group_size=group_size,
sym=sym,
iters=2,
seqlen=2,
dataset=dataloader,
)
quantized_model_path = self.save_folder
_, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
model = AutoModelForCausalLM.from_pretrained(
quantized_model_path, device_map="auto", trust_remote_code=True
)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])
print(res)
assert "!!!" not in res