PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix dtype converting issue (#403)
Wenhua Cheng6ac7a2b2cc6c
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex 592506a46..376bc3e32 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -285,8 +285,6 @@ class AutoRoundQuantizer(HfQuantizer):def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":if torch_dtype is None:torch_dtype = torch.float16- elif torch_dtype != torch.float16 and not is_hpu_supported():- logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with AutoRound.")return torch_dtypedef find_backend(self, target_backend: str):@@ -406,10 +404,9 @@ class AutoRoundQuantizer(HfQuantizer):if ("hpu" == target_device or "cpu" == target_device) and model.dtype != torch.bfloat16:logger.info(f"Change the dtype to `bfloat16` as {target_device.upper()} does not support float16")model = model.to(torch.bfloat16)- else:- if model.dtype != torch.float16:- logger.info(f"Change the dtype to `float16` for better performance")- model = model.to(torch.float16)+ elif "cuda" == target_device and model.dtype != torch.float16:+ logger.info(f"Change the dtype to `float16` for better performance")+ model = model.to(torch.float16)bits = quantization_config.bitsgroup_size = quantization_config.group_size