PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
little change for autogptq exporting
wenhuach21c97a68e14aee
diff --git a/README.md b/README.mdindex 7ac464011..87d4852dc 100644--- a/README.md+++ b/README.md@@ -46,8 +46,9 @@ autoround = AutoRound(model, tokenizer, bits=4, group_size=128, scheme="asym")fake_qdq_model,weight_config = autoround.quantize() ##scale,zp info are saved in weight config dict## export to gpu+## please install autogptq first# packed_folder = "./tmp_autoround_packed"-# autoround.export_to_autogptq(packed_folder)+# autoround.export_to_autogptq(packed_folder, use_triton=True) ## Utilizing Triton for 2-bit and 4-bit scenarios```### Detailed Hyperparametersdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 4a5c95146..ba0e6535e 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1342,7 +1342,7 @@ class AutoRound(object):for i in range(0, len(block_names), n_blocks):if n_blocks == 1:n = block_names[i]- logger.info(f"quantizing {i+1}/{len(block_names)}, {n}")+ logger.info(f"quantizing {i + 1}/{len(block_names)}, {n}")m = get_module(model, n)else:names = block_names[i: i + n_blocks]@@ -1369,7 +1369,7 @@ class AutoRound(object):torch.cuda.empty_cache()- def export_to_autogptq(self, output_dir):+ def export_to_autogptq(self, output_dir, use_triton=False):"""Export the model to autogptq format to easily leverage cuda kernel"""@@ -1379,7 +1379,9 @@ class AutoRound(object):model = copy.deepcopy(self.model.to("cpu")) ##TODO avoid this deepcopyfrom auto_gptq.modeling._utils import pack_model- if self.bits == 3:+ if self.bits == 3 or use_triton is False:+ if self.bits == 3 and use_triton is True:+ logger.warning("triton does not support 3 bits, reset it to False")quantizers = {}for key in self.weight_config:info = self.weight_config[key]@@ -1387,7 +1389,7 @@ class AutoRound(object):continuequantizers[key] = (None, info['scale'], info['zp'], info['g_idx'])pack_model(model, quantizers, self.bits, self.group_size, use_cuda_fp16=True, desc_act=False,- force_layer_back_to_cpu=True)+ force_layer_back_to_cpu=True, use_triton=False)else:quantizers = {}for key in self.weight_config: