Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix dtype converting issue (#403)

Wenhua Cheng6ac7a2b2cc6c
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.py
index 592506a46..376bc3e32 100644
--- a/auto_round/auto_quantizer.py
+++ b/auto_round/auto_quantizer.py
@@ -285,8 +285,6 @@ class AutoRoundQuantizer(HfQuantizer):
def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
if torch_dtype is None:
torch_dtype = torch.float16
- elif torch_dtype != torch.float16 and not is_hpu_supported():
- logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with AutoRound.")
return torch_dtype
def find_backend(self, target_backend: str):
@@ -406,10 +404,9 @@ class AutoRoundQuantizer(HfQuantizer):
if ("hpu" == target_device or "cpu" == target_device) and model.dtype != torch.bfloat16:
logger.info(f"Change the dtype to `bfloat16` as {target_device.upper()} does not support float16")
model = model.to(torch.bfloat16)
- else:
- if model.dtype != torch.float16:
- logger.info(f"Change the dtype to `float16` for better performance")
- model = model.to(torch.float16)
+ elif "cuda" == target_device and model.dtype != torch.float16:
+ logger.info(f"Change the dtype to `float16` for better performance")
+ model = model.to(torch.float16)
bits = quantization_config.bits
group_size = quantization_config.group_size