Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix awq exporting at group_size=-1 (#579)

Wenhua Cheng2bb57c23c88e
diff --git a/README.md b/README.md
index 54207c37f..81436d9aa 100644
--- a/README.md
+++ b/README.md
@@ -345,18 +345,18 @@ Triton, but the final choice depends on factors such as bits, group_size, packin
backend may not always be the most suitable for certain devices. Please refer
to the following table for the details and specify the backend you want.
-| Name | Devices | Bits | Dtypes | Priority | Packing format | Requirements |
-|--------------------------------------|----------|---------|-----------|----------|-----------------|----------------------------------|
-| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
-| itrex | cpu | 2,4,8 | BF16/FP16 | 0 | gptq_zp+-1/awq | intel-extension-for-transformers |
-| marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
-| exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |
-| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |
-| gptq:cuda | cuda | 2,3,4,8 | FP16 | 0 | gptq_zp+-1 | auto-gptq |
-| triton | cuda/xpu | 2,4,8 | BF16/FP16 | 1 | gptq/gptq_zp+-1 | auto-round |
-| awq | cuda | 4 | FP16 | 5 | awq | auto-awq |
-| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |
-| torch | cuda/xpu/cpu | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
+| Name | Devices | Bits | Dtypes | Priority | Packing format | Requirements |
+|--------------------------------------|--------------|---------|-----------|----------|-----------------|----------------------------------|
+| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
+| itrex | cpu | 2,4,8 | BF16/FP16 | 1 | gptq_zp+-1/awq | <br/>intel-extension-for-transformers |
+| marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
+| exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |
+| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |
+| gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq |
+| triton | xpu/cuda | 2,4,8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | auto-round |
+| awq | cuda | 4 | FP16 | 5 | awq | auto-awq |
+| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |
+| torch | xpu/cpu/cuda | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
```python
from transformers import AutoModelForCausalLM, AutoTokenizer
diff --git a/auto_round/export/export_to_awq/utils.py b/auto_round/export/export_to_awq/utils.py
index 48b00ccc1..64e826e9e 100644
--- a/auto_round/export/export_to_awq/utils.py
+++ b/auto_round/export/export_to_awq/utils.py
@@ -219,10 +219,10 @@ class WQLinear_GEMM(nn.Module):
device = "cuda:0"
elif torch.xpu.is_available():
device = "xpu:0"
-
- repeat_scales = scales.to(device).t().repeat_interleave(group_size, 1)
+ repeat_size = group_size if group_size != -1 else linear.in_features
+ repeat_scales = scales.to(device).t().repeat_interleave(repeat_size, 1)
if isinstance(zeros, torch.Tensor):
- repeat_zeros = zeros.to(device).t().repeat_interleave(group_size, 1)
+ repeat_zeros = zeros.to(device).t().repeat_interleave(repeat_size, 1)
intweight = torch.round(
linear.weight.to(device) / repeat_scales[:, :linear.weight.shape[1]] +
repeat_zeros[:, :linear.weight.shape[1]]).to(torch.int).t().contiguous()
diff --git a/docs/step_by_step.md b/docs/step_by_step.md
index 725b4decb..c70f3f8af 100644
--- a/docs/step_by_step.md
+++ b/docs/step_by_step.md
@@ -481,15 +481,15 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50, do_sample=Fal
| Name | Devices | Bits | Dtypes | Priority | Packing format | Requirements |
|--------------------------------------|----------|---------|-----------|----------|-----------------|-------------------------------|
| ipex | cpu/xpu | 4 | BF16/FP16 | 5 | gptq_zp+-1/awq | intel-extension-for-pytorch |
-| itrex | cpu | 2,4,8 | BF16/FP16 | 0 | gptq_zp+-1/awq | intel-extension-for-transformers |
+| itrex | cpu | 2,4,8 | BF16/FP16 | 1 | gptq_zp+-1/awq | <br/>intel-extension-for-transformers |
| marlin | cuda | 4,8 | BF16/FP16 | 6 | gptq/gptq_zp+-1 | gptqmodel |
| exllamav2 or<br/>gptqmodel:exllamav2 | cuda | 4 | BF16/FP16 | 5 | gptq | gptqmodel |
| exllamav2 or<br/>gptq:exllamav2 | cuda | 4 | FP16 | 5 | gptq_zp+-1 | auto-gptq |
-| gptq:cuda | cuda | 2,3,4,8 | FP16 | 0 | gptq_zp+-1 | auto-gptq |
-| triton | cuda/xpu | 2,4,8 | BF16/FP16 | 1 | gptq/gptq_zp+-1 | auto-round |
+| gptq:cuda | cuda | 2,3,4,8 | FP16 | 1 | gptq_zp+-1 | auto-gptq <br/> |
+| triton | xpu/cuda | 2,4,8 | BF16/FP16 | 2 | gptq/gptq_zp+-1 | <br/>auto-round |
| awq | cuda | 4 | FP16 | 5 | awq | auto-awq |
| hpu | hpu | 4 | BF16 | 0 | gptq/gptq_zp+-1 | auto-round |
-| torch | cuda/xpu/cpu | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
+| torch | xpu/cpu/cuda | 2,3,4,8 | BF16/FP16 | 0 | gptq/gptq_zp+-1 | auto-round |
### Convert GPTQ/AWQ to AutoRound
diff --git a/test/test_export.py b/test/test_export.py
index c1eb40268..b590b7905 100644
--- a/test/test_export.py
+++ b/test/test_export.py
@@ -29,119 +29,129 @@ class TestAutoRound(unittest.TestCase):
@classmethod
def tearDownClass(self):
- shutil.rmtree(self.save_dir, ignore_errors=True)
+ shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
def test_autogptq_format(self):
- if not torch.cuda.is_available():
- return
- bits, group_size, sym = 4, 128, False
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- autoround.quantize()
- quantized_model_path = self.save_dir
-
- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- shutil.rmtree(quantized_model_path, ignore_errors=True)
+ for group_size in [-1, 32, 128]:
+ bits, sym = 4, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = "./saved"
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_gptq")
+
+ if group_size == -1:
+ shutil.rmtree("./saved", ignore_errors=True)
+ continue
+ quantization_config = AutoRoundConfig(
+ )
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu",
+ trust_remote_code=True,
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
def test_autoround_format(self):
- bits, group_size, sym = 4, 128, True
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- autoround.quantize()
- quantized_model_path = self.save_dir
+ for group_size in [-1, 32, 128]:
+ bits, sym = 4, True
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = "./saved"
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
+
+ if group_size == -1:
+ shutil.rmtree("./saved", ignore_errors=True)
+ continue
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
-
-
- device = "auto" ##cpu, hpu, cuda
- quantization_config = AutoRoundConfig(
- backend=device
- )
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,
- quantization_config=quantization_config)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- shutil.rmtree(quantized_model_path, ignore_errors=True)
def test_autoround_awq_format(self):
- bits, group_size, sym = 4, 128, False
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- autoround.quantize()
- quantized_model_path = self.save_dir
-
- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:awq")
-
- quantization_config = AutoRoundConfig(
- backend="cpu"
- )
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu",
- quantization_config=quantization_config)
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- shutil.rmtree(quantized_model_path, ignore_errors=True)
+ for group_size in [-1, 32, 128]:
+ bits, sym = 4, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = "./saved"
+
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round:auto_awq")
+
+ # quantization_config = AutoRoundConfig(
+ # backend="cpu"
+ # )
+ if group_size == -1:
+ continue
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu")
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
def test_autoawq_format(self):
- bits, group_size, sym = 4, 128, False
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- autoround.quantize()
- quantized_model_path = self.save_dir
-
- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \
- format="auto_awq", model_path="facebook/opt-125m")
- try:
- import awq
- except:
- return
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- text = "There is a girl who likes adventure,"
- inputs = tokenizer(text, return_tensors="pt").to(model.device)
- print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- shutil.rmtree(quantized_model_path, ignore_errors=True)
+ for group_size in [-1, 32, 128]:
+ bits, sym = 4, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+ quantized_model_path = "./saved"
+
+ autoround.save_quantized(output_dir=quantized_model_path, inplace=False, \
+ format="auto_awq")
+ if group_size == -1:
+ shutil.rmtree("./saved", ignore_errors=True)
+ continue
+ quantization_config = AutoRoundConfig()
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cpu",
+ quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ shutil.rmtree("./saved", ignore_errors=True)
def test_autoround_3bit_asym_format(self):
@@ -185,7 +195,7 @@ class TestAutoRound(unittest.TestCase):
quantized_model_path = self.save_dir
autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="auto_round")
- device = "auto" ##cpu, hpu, cuda
+ device = "cpu" ##cpu, hpu, cuda
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"