PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix export issue with torch 2.0 (#129)
wenhuach215bff86eedad9
diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.pyindex 06f4694b8..6ef2a08c8 100644--- a/auto_round/export/export_to_autogptq.py+++ b/auto_round/export/export_to_autogptq.py@@ -107,7 +107,8 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarinfo = weight_config[key]if not check_to_quantized(info):continue- quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])+ ##force to float32 to be compatible with torch 2.0+ quantizers[key] = (None, info["scale"], info["zp"].to(torch.float32), info["g_idx"])pack_model(compressed_model,quantizers,@@ -126,7 +127,7 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwarinfo = weight_config[key]if not check_to_quantized(info):continue- quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"])+ quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"].to(torch.float32), info["g_idx"])pack_model(compressed_model,quantizers,@@ -158,20 +159,20 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwardef _save_quantized_to_autogptq(- model,- save_dir: str,- bits=4,- group_size=128,- sym=False,- iters=200,- lr=5e-3,- minmax_lr=5e-3,- enable_minmax_tuning=True,- enable_quanted_input=True,- use_safetensors: bool = True,- scale_dtype=torch.float32,- safetensors_metadata: Optional[Dict[str, str]] = None,- modules_in_block_to_quantize=None,+ model,+ save_dir: str,+ bits=4,+ group_size=128,+ sym=False,+ iters=200,+ lr=5e-3,+ minmax_lr=5e-3,+ enable_minmax_tuning=True,+ enable_quanted_input=True,+ use_safetensors: bool = True,+ scale_dtype=torch.float32,+ safetensors_metadata: Optional[Dict[str, str]] = None,+ modules_in_block_to_quantize=None,):"""Save quantized model and configs to local disk for cuda."""os.makedirs(save_dir, exist_ok=True)diff --git a/auto_round/export/export_to_autoround/export_to_autoround.py b/auto_round/export/export_to_autoround/export_to_autoround.pyindex 34d0ac132..14f071f38 100644--- a/auto_round/export/export_to_autoround/export_to_autoround.py+++ b/auto_round/export/export_to_autoround/export_to_autoround.py@@ -151,7 +151,8 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="gptq:exllamavzero = weight_config[name]["zp"]# so far can only pack layer on CPUqlayer.to("cpu")- layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu")+ ##force to float32 to be compatible with torch 2.0+ layer, scale, zero = layer.to("cpu"), scale.to("cpu"), zero.to("cpu").to(torch.float32)qlayer.pack(layer, scale, zero, None)qlayer.to(device)quantization_config = kwargs["serialization_dict"]