PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
ix incorrect device setting in autoround format inference (#383)
WeiweiZhang1e88882ef78df
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex 081274d99..e3a193e2f 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -363,7 +363,14 @@ class AutoRoundQuantizer(HfQuantizer):if backend is None:raise ValueError("Backend not found, please set it to 'auto' to have a try ")- return BackendInfos[backend].device[0]+ device = BackendInfos[backend].device[0]+ if "cuda" in device and torch.cuda.is_available():+ return device+ elif "hpu" in device and is_hpu_supported():+ return device+ else:+ return "cpu"+def convert_model(self, model: nn.Module):"""Converts the given model to an AutoRound model by replacing its layers with quantized layers.@@ -392,6 +399,7 @@ class AutoRoundQuantizer(HfQuantizer):quantization_config.target_backend = quantization_config.backendtarget_device = self.detect_device(quantization_config.target_backend, quantization_config.backend)+self.target_device = target_deviceif hasattr(quantization_config, "backend"): # pragma: no cover@@ -744,3 +752,4 @@ if version.parse(transformers.__version__) < version.parse("4.38.0"):transformers.quantizers.auto.AutoHfQuantizer = AutoHfQuantizertransformers.modeling_utils.AutoHfQuantizer = AutoHfQuantizer+