PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bug of only_text_test check due to inference issue on cpu (#362)
Heng Guo3acb11915d53
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 42993bef9..61f89338f 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -30,13 +30,15 @@ from .mllm_dataset import get_mllm_dataloaderfrom ..low_cpu_mem.utils import get_layers_before_block-def _only_text_test(model, tokenizer):+def _only_text_test(model, tokenizer, device):"""Test if the model whether can use text-only datasets."""try:text = ["only text", "test"]tokenizer.padding_side = 'left'if tokenizer.pad_token is None:tokenizer.pad_token = tokenizer.eos_token+ if device != model.device.type:+ model = model.to(device)inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True).to(model.device)model(**inputs)return True@@ -155,7 +157,8 @@ class AutoRoundMLLM(AutoRound):from ..calib_dataset import CALIB_DATASETSfrom .mllm_dataset import MLLM_DATASETif isinstance(dataset, str):- if quant_nontext_module or (dataset in CALIB_DATASETS.keys() and not _only_text_test(model, tokenizer)):+ if quant_nontext_module or \+ (dataset in CALIB_DATASETS.keys() and not _only_text_test(model, tokenizer, device)):if quant_nontext_module:logger.warning(f"Text only dataset cannot be used for calibrating non-text modules,""switching to liuhaotian/llava_conv_58k")diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 4e4a10cba..cf373b792 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -45,10 +45,12 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--eval_bs", default=None, type=int,help="batch size in evaluation")- self.add_argument("--device", "--devices", default="auto", type=str,- help="the device to be used for tuning. The default is set to auto,"- "allowing for automatic detection."- "Currently, device settings support CPU, GPU, and HPU.")+ self.add_argument("--device", "--devices", default="0", type=str,+ help="the device to be used for tuning. "+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.")self.add_argument("--asym", action='store_true',help="whether to use asym quantization")@@ -268,6 +270,8 @@ def tune(args):devices = args.device.replace(" ", "").split(',')if len(devices) > 1: ##for 70B model on single card, use auto will cause some layer offload to cpuuse_auto_mapping = True+ elif args.device == "auto":+ use_auto_mapping == Trueimport reimport torchdiff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 825bf1c84..54d35d474 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -44,10 +44,12 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--eval_bs", default=None, type=int,help="batch size in evaluation")- self.add_argument("--device", "--devices", default="auto", type=str,- help="the device to be used for tuning. The default is set to auto,"- "allowing for automatic detection."- "Currently, device settings support CPU, GPU, and HPU.")+ self.add_argument("--device", "--devices", default="0", type=str,+ help="the device to be used for tuning. "+ "Currently, device settings support CPU, GPU, and HPU."+ "The default is set to cuda:0,"+ "allowing for automatic detection and switch to HPU or CPU."+ "set --device 0,1,2 to use multiple cards.")self.add_argument("--asym", action='store_true',help="whether to use asym quantization")@@ -269,6 +271,8 @@ def tune(args):args.device = ",".join(map(str, range(len(devices))))devices = args.device.replace(" ", "").split(',')use_auto_mapping = True+ elif args.device == "auto":+ use_auto_mapping == Truedevice_str = detect_device(devices[0])