PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix awq exporting at group_size=-1 (#579)
Wenhua Cheng2bb57c23c88e
diff --git a/README.md b/README.mdindex 54207c37f..81436d9aa 100644--- a/README.md+++ b/README.md@@ -345,18 +345,18 @@ Triton, but the final choice depends on factors such as bits, group_size, packinbackend may not always be the most suitable for certain devices. Please referto the following table for the details and specify the backend you want.-| Name | Devices | Bits | Dtypes | Priority | Packing format | Requirements |-|--------------------------------------|----------|---------|-----------|----------|-----------------|----------------------------------|-| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |-| itrex | cpu | 2,4,8 | BF16/FP16 | 0 | gptq_zp+-1/awq | intel-extension-for-transformers |-| marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |-| exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |-| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |-| gptq:cuda | cuda | 2,3,4,8 | FP16 | 0 | gptq_zp+-1 | auto-gptq |-| triton | cuda/xpu | 2,4,8 | BF16/FP16 | 1 | gptq/gptq_zp+-1 | auto-round |-| awq | cuda | 4 | FP16 | 5 | awq | auto-awq |-| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |-| torch | cuda/xpu/cpu | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |+| Name | Devices | Bits | Dtypes | Priority | Packing format | Requirements |+|--------------------------------------|--------------|---------|-----------|----------|-----------------|----------------------------------|+| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |+| itrex | cpu | 2,4,8 | BF16/FP16 | 1 | gptq_zp+-1/awq | <br/>intel-extension-for-transformers |+| marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |+| exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |+| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |+| gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq |+| triton | xpu/cuda | 2,4,8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | auto-round |+| awq | cuda | 4 | FP16 | 5 | awq | auto-awq |+| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |+| torch | xpu/cpu/cuda | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizerdiff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.pyindex 48b00ccc1..64e826e9e 100644--- a/auto_round/export/export_to_awq/utils.py+++ b/auto_round/export/export_to_awq/utils.py@@ -219,10 +219,10 @@ class WQLinear_GEMM(nn.Module):device = "cuda:0"elif torch.xpu.is_available():device = "xpu:0"-- repeat_scales = scales.to(device).t().repeat_interleave(group_size, 1)+ repeat_size = group_size if group_size != -1 else linear.in_features+ repeat_scales = scales.to(device).t().repeat_interleave(repeat_size, 1)if isinstance(zeros, torch.Tensor):- repeat_zeros = zeros.to(device).t().repeat_interleave(group_size, 1)+ repeat_zeros = zeros.to(device).t().repeat_interleave(repeat_size, 1)intweight = torch.round(linear.weight.to(device) / repeat_scales[:, :linear.weight.shape[1]] +repeat_zeros[:, :linear.weight.shape[1]]).to(torch.int).t().contiguous()diff --git a/docs/step_by_step.md b/docs/step_by_step.mdindex 725b4decb..c70f3f8af 100644--- a/docs/step_by_step.md+++ b/docs/step_by_step.md@@ -481,15 +481,15 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal| Name | Devices | Bits | Dtypes | Priority | Packing format | Requirements ||--------------------------------------|----------|---------|-----------|----------|-----------------|-------------------------------|| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |-| itrex | cpu | 2,4,8 | BF16/FP16 | 0 | gptq_zp+-1/awq | intel-extension-for-transformers |+| itrex | cpu | 2,4,8 | BF16/FP16 | 1 | gptq_zp+-1/awq | <br/>intel-extension-for-transformers || marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel || exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel || exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |-| gptq:cuda | cuda | 2,3,4,8 | FP16 | 0 | gptq_zp+-1 | auto-gptq |-| triton | cuda/xpu | 2,4,8 | BF16/FP16 | 1 | gptq/gptq_zp+-1 | auto-round |+| gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq <br/> |+| triton | xpu/cuda | 2,4,8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | <br/>auto-round || awq | cuda | 4 | FP16 | 5 | awq | auto-awq || hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |-| torch | cuda/xpu/cpu | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |+| torch | xpu/cpu/cuda | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |### Convert GPTQ/AWQ to AutoRounddiff --git a/test/test_export.py b/test/test_export.pyindex c1eb40268..b590b7905 100644--- a/test/test_export.py+++ b/test/test_export.py@@ -29,119 +29,129 @@ class TestAutoRound(unittest.TestCase):@classmethoddef tearDownClass(self):- shutil.rmtree(self.save_dir, ignore_errors=True)+ shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)def test_autogptq_format(self):- if not torch.cuda.is_available():- return- bits, group_size, sym = 4, 128, False- autoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- autoround.quantize()- quantized_model_path = self.save_dir-- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- shutil.rmtree(quantized_model_path, ignore_errors=True)+ for group_size in [-1, 32, 128]:+ bits, sym = 4, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = "./saved"+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")++ if group_size == -1:+ shutil.rmtree("./saved", ignore_errors=True)+ continue+ quantization_config = AutoRoundConfig(+ )+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu",+ trust_remote_code=True,+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree("./saved", ignore_errors=True)def test_autoround_format(self):- bits, group_size, sym = 4, 128, True- autoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- autoround.quantize()- quantized_model_path = self.save_dir+ for group_size in [-1, 32, 128]:+ bits, sym = 4, True+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = "./saved"+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")++ if group_size == -1:+ shutil.rmtree("./saved", ignore_errors=True)+ continue+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree("./saved", ignore_errors=True)- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")--- device = "auto" ##cpu, hpu, cuda- quantization_config = AutoRoundConfig(- backend=device- )- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,- quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- shutil.rmtree(quantized_model_path, ignore_errors=True)def test_autoround_awq_format(self):- bits, group_size, sym = 4, 128, False- autoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- autoround.quantize()- quantized_model_path = self.save_dir-- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:awq")-- quantization_config = AutoRoundConfig(- backend="cpu"- )- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu",- quantization_config=quantization_config)- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- shutil.rmtree(quantized_model_path, ignore_errors=True)+ for group_size in [-1, 32, 128]:+ bits, sym = 4, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = "./saved"++ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:auto_awq")++ # quantization_config = AutoRoundConfig(+ # backend="cpu"+ # )+ if group_size == -1:+ continue++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree("./saved", ignore_errors=True)def test_autoawq_format(self):- bits, group_size, sym = 4, 128, False- autoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- autoround.quantize()- quantized_model_path = self.save_dir-- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \- format="auto_awq", model_path="facebook/opt-125m")- try:- import awq- except:- return- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)- text = "There is a girl who likes adventure,"- inputs = tokenizer(text, return_tensors="pt").to(model.device)- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))- shutil.rmtree(quantized_model_path, ignore_errors=True)+ for group_size in [-1, 32, 128]:+ bits, sym = 4, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()+ quantized_model_path = "./saved"++ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \+ format="auto_awq")+ if group_size == -1:+ shutil.rmtree("./saved", ignore_errors=True)+ continue+ quantization_config = AutoRoundConfig()++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu",+ quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ shutil.rmtree("./saved", ignore_errors=True)def test_autoround_3bit_asym_format(self):@@ -185,7 +195,7 @@ class TestAutoRound(unittest.TestCase):quantized_model_path = self.save_dirautoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")- device = "auto" ##cpu, hpu, cuda+ device = "cpu" ##cpu, hpu, cudamodel = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"