PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
api support for fp8 model and mllm api support load from str (#752)
Heng Guofcbcaa48de52
diff --git a/README.md b/README.mdindex a23785a0a..819167905 100644--- a/README.md+++ b/README.md@@ -300,7 +300,7 @@ for output in outputs:### Transformers (CPU/Intel GPU/Gaudi/CUDA)-AutoRound support 10+ backends an automatically selects the best available backend based on the installed libraries and prompts the user to+AutoRound support 10+ backends and automatically selects the best available backend based on the installed libraries and prompts the user toinstall additional libraries when a better backend is found.**Please avoid manually moving the quantized model to a different device** (e.g., model.to('cpu')) during inference, asdiff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 15ee357b5..aedcd860d 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -40,6 +40,7 @@ from auto_round.utils import (CpuInfo,_gguf_args_check,block_forward,+ check_and_mark_fp8_model,check_is_cpu,check_need_act_calibration,check_seqlen_compatible,@@ -223,11 +224,31 @@ class AutoRound(object):os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"torch.use_deterministic_algorithms(True, warn_only=False)+ # Model related+ self.quantized = False+ if isinstance(model, str):+ model, tokenizer, low_cpu_mem_usage = llm_load_model(+ model, device=device, low_cpu_mem_mode=low_cpu_mem_usage+ )+ elif tokenizer is None and iters > 0:+ raise ValueError("A tokenizer must be set for non-str model input")+ self.low_cpu_mem_usage = bool(low_cpu_mem_usage)+ if unsupport_meta_device(model):+ raise RuntimeError(+ "AutoRound does not support parameters on meta device. "+ "Please use more GPUs by setting `--device 0,1,2,3` or just place the model on CPU."+ )+ check_and_mark_fp8_model(model)+ model = _handle_moe_model(model)+ self.model = model.eval()+ self.tokenizer = tokenizer+ self.shared_cache_keys = get_shared_keys(self.model)+# Set quantization configsself.bits = bitsself.data_type = data_typetmp_bits = infer_bits_by_data_type(self.data_type)- if tmp_bits < 16 and tmp_bits != bits:+ if tmp_bits is not None and tmp_bits < 16 and tmp_bits != bits:logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")self.bits = tmp_bitsself.group_size = group_size@@ -248,7 +269,7 @@ class AutoRound(object):else:self.act_data_type = "float"tmp_act_bits = infer_bits_by_data_type(self.act_data_type)- if tmp_act_bits < 16 and tmp_act_bits != self.act_bits:+ if tmp_act_bits is not None and tmp_act_bits < 16 and tmp_act_bits != self.act_bits:self.act_bits = tmp_act_bitslogger.warning(f"`act_data_type` do not"@@ -1711,14 +1732,14 @@ class AutoRound(object):layer_config[n]["fixed_by_user"] = True# Check dtype and act_dtypetmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])- if tmp_bits != layer_config[n]["bits"]:+ if tmp_bits is not None and tmp_bits != layer_config[n]["bits"]:logger.warning(f"'data_type' do not match the specified 'bits' setting for {n}."f" Resetting 'bits' to {tmp_bits}.")layer_config[n]["bits"] = tmp_bitstmp_act_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])- if tmp_act_bits != layer_config[n]["act_bits"]:+ if tmp_act_bits is not None and tmp_act_bits != layer_config[n]["act_bits"]:logger.warning(f"'act_data_type' do not match the specified 'act_bits' setting for {n}."f" Resetting 'act_bits' to {tmp_act_bits}."diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 67b64e113..90532aa21 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -18,15 +18,16 @@ from typing import Unionimport torchfrom tqdm import tqdm+from auto_round.autoround import AutoRound+from auto_round.low_cpu_mem.utils import get_layers_before_block+from auto_round.mllm.mllm_dataset import get_mllm_dataloader+from auto_round.mllm.template import Template, get_templatefrom auto_round.special_model_handler import (NOT_SUPPORT_ONLY_TEXT_MODELS,SUPPORT_ONLY_TEXT_MODELS,_handle_special_model,)--from ..autoround import AutoRound-from ..low_cpu_mem.utils import get_layers_before_block-from ..utils import (+from auto_round.utils import (check_to_quantized,clear_memory,detect_device,@@ -38,8 +39,6 @@ from ..utils import (to_device,to_dtype,)-from .mllm_dataset import get_mllm_dataloader-from .template import Template, get_templatedef _only_text_test(model, tokenizer, device, model_type):@@ -129,8 +128,8 @@ class AutoRoundMLLM(AutoRound):def __init__(self,- model: torch.nn.Module,- tokenizer,+ model: Union[torch.nn.Module, str],+ tokenizer=None,processor=None,image_processor=None,bits: int = 4,@@ -139,7 +138,7 @@ class AutoRoundMLLM(AutoRound):layer_config: dict = None,batch_size: int = 8,amp: bool = True,- device: str = None,+ device: Union[str, torch.device, int] = 0,lr_scheduler=None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = None,extra_data_dir: str = None,@@ -173,6 +172,9 @@ class AutoRoundMLLM(AutoRound):model_kwargs: dict = None,**kwargs,):+ if isinstance(model, str):+ model, processor, tokenizer, image_processor = mllm_load_model(model, device=device)+quant_nontext_module = self._check_quant_nontext(layer_config, quant_nontext_module)all_blocks = get_block_names(model, quant_nontext_module)self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 76177e5bb..b24bc5bd6 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -628,7 +628,8 @@ def tune(args):import timeeval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")- if args.act_bits <= 8 or infer_bits_by_data_type(args.act_data_type) <= 8 or eval_gguf_model:+ tmp_act_bits = infer_bits_by_data_type(args.act_data_type)+ if tmp_act_bits is not None and (args.act_bits <= 8 or tmp_act_bits <= 8 or eval_gguf_model):if eval_gguf_model:# for file in os.listdir(eval_folder):# gguf_file = filediff --git a/auto_round/utils.py b/auto_round/utils.pyindex 635648299..40fbcc5de 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -97,7 +97,7 @@ def infer_bits_by_data_type(data_type: str):return int(suc_2str)if str.isdigit(data_type[len(supported_dtype)]):return int(data_type[len(supported_dtype)])- return 16+ return None@lru_cache(None)@@ -1116,15 +1116,15 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Nonreturn True, ""-def get_device_and_parallelism(device):+def get_device_and_parallelism(device: Union[str, torch.device, int]) -> Tuple[str, bool]:from auto_round.utils import detect_deviceif isinstance(device, str):devices = device.replace(" ", "").split(",")elif isinstance(device, int):- devices = str(device)+ devices = [str(device)]else:- devices = device+ devices = [device]if all(s.isdigit() for s in devices) and len(devices) > 1 and torch.cuda.is_available():device = "cuda"parallelism = Truediff --git a/test/test_cpu/test_mllm.py b/test/test_cpu/test_mllm.pyindex cd9fb25b4..0fd102407 100644--- a/test/test_cpu/test_mllm.py+++ b/test/test_cpu/test_mllm.py@@ -41,16 +41,9 @@ class TestAutoRoundMLLM(unittest.TestCase):return super().tearDownClass()def test_tune(self):- tokenizer = AutoTokenizer.from_pretrained(self.model_name)- processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)- model = Qwen2VLForConditionalGeneration.from_pretrained(- self.model_name, trust_remote_code=True, device_map="auto"- )bits, group_size = 4, 128autoround = AutoRoundMLLM(- model,- tokenizer,- processor=processor,+ model=self.model_name,bits=bits,group_size=group_size,nsamples=1,