PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix device auto issue (#400)
Wenhua Cheng73fcf7d7701f
diff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex e3a193e2f..638032a5c 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -501,6 +501,8 @@ class AutoRoundQuantizer(HfQuantizer):target_backend = remove_device_str(target_backend, "hpu")target_backend = remove_device_str(target_backend, "cuda")orig_backend = self.find_backend(orig_backend)+ if orig_backend is None:+ orig_backend = "gptq:exllamav2"if target_backend == "":target_backend = orig_backenddiff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 62beefc1d..5ecb93e3b 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -298,7 +298,7 @@ def tune(args):if len(devices) > 1: ##for 70B model on single card, use auto will cause some layer offload to cpuuse_auto_mapping = Trueelif args.device == "auto":- use_auto_mapping == True+ use_auto_mapping = Trueimport reimport torchdiff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex a3a3dd9fc..5bc0203be 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -271,7 +271,7 @@ def tune(args):if len(devices) > 1:use_auto_mapping = True ##for 70B model on single card, use auto will cause some layer offload to cpuelif args.device == "auto":- use_auto_mapping == True+ use_auto_mapping = Truedevice_str = detect_device(devices[0])