Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

little change for autogptq exporting

wenhuach21c97a68e14aee
diff --git a/README.md b/README.md
index 7ac464011..87d4852dc 100644
--- a/README.md
+++ b/README.md
@@ -46,8 +46,9 @@ autoround = AutoRound(model, tokenizer, bits=4, group_size=128, scheme="asym")
fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in weight config dict
## export to gpu
+## please install autogptq first
# packed_folder = "./tmp_autoround_packed"
-# autoround.export_to_autogptq(packed_folder)
+# autoround.export_to_autogptq(packed_folder, use_triton=True) ## Utilizing Triton for 2-bit and 4-bit scenarios
```
### Detailed Hyperparameters
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 4a5c95146..ba0e6535e 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1342,7 +1342,7 @@ class AutoRound(object):
for i in range(0, len(block_names), n_blocks):
if n_blocks == 1:
n = block_names[i]
- logger.info(f"quantizing {i+1}/{len(block_names)}, {n}")
+ logger.info(f"quantizing {i + 1}/{len(block_names)}, {n}")
m = get_module(model, n)
else:
names = block_names[i: i + n_blocks]
@@ -1369,7 +1369,7 @@ class AutoRound(object):
torch.cuda.empty_cache()
- def export_to_autogptq(self, output_dir):
+ def export_to_autogptq(self, output_dir, use_triton=False):
"""
Export the model to autogptq format to easily leverage cuda kernel
"""
@@ -1379,7 +1379,9 @@ class AutoRound(object):
model = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopy
from auto_gptq.modeling._utils import pack_model
- if self.bits == 3:
+ if self.bits == 3 or use_triton is False:
+ if self.bits == 3 and use_triton is True:
+ logger.warning("triton does not support 3 bits, reset it to False")
quantizers = {}
for key in self.weight_config:
info = self.weight_config[key]
@@ -1387,7 +1389,7 @@ class AutoRound(object):
continue
quantizers[key] = (None, info['scale'], info['zp'], info['g_idx'])
pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,
- force_layer_back_to_cpu=True)
+ force_layer_back_to_cpu=True, use_triton=False)
else:
quantizers = {}
for key in self.weight_config: