PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support model_dtype and fix bug of scheme contains quotes, mllm eval (#985)
Heng Guocd308ddb7616
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex c4b2683f5..76a8f73d1 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -52,6 +52,7 @@ class BasicArgumentParser(argparse.ArgumentParser):help="Path to the pre-trained model or model identifier from huggingface.co/models. ""Examples: 'facebook/opt-125m', 'bert-base-uncased', or local path like '/path/to/model'",)+ basic.add_argument("--model_dtype", default=None, help="model dtype used to load the pre-trained model")basic.add_argument("--platform",default="hf",@@ -589,6 +590,7 @@ def tune(args):enable_adam=args.adam,extra_config=extra_config,layer_config=layer_config,+ model_dtype=args.model_dtype,)model_name = args.model.rstrip("/")@@ -765,6 +767,7 @@ def tune(args):batch_size=args.eval_bs,limit=args.limit,eval_model_dtype=eval_model_dtype,+ mllm=autoround.mllm, # pylint: disable=E1101)else:from auto_round.eval.evaluation import simple_evaluate@@ -775,8 +778,15 @@ def tune(args):st = time.time()if "llama" in args.model.lower():model_args += ",add_bos_token=True"+ if autoround.mllm: # pylint: disable=E1101+ model_type = "hf-multimodal"+ if args.eval_bs is None or args.eval_bs == "auto":+ logger.warning("hf-multimodal models does not support auto currently, reset eval_bs to 16")+ args.eval_bs = 16+ else:+ model_type = "hf"res = simple_evaluate(- model="hf",+ model=model_type,model_args=model_args,tasks=tasks,device=device_str,@@ -794,10 +804,15 @@ def setup_eval_parser():def run_eval():+ from auto_round.utils import is_mllm_model+args = setup_eval_parser()assert args.model or args.model_name, "[model] or --model MODEL_NAME should be set."if args.model is None:args.model = args.model_name+ if is_mllm_model(args.model):+ args.mllm = True+if args.eval_task_by_task:eval_task_by_task(model=args.model,diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex ff06ec851..d52914447 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -120,6 +120,7 @@ class AutoRound:device_map (str | dict, optional): Device placement map. Defaults to None.disable_opt_rtn (bool, optional): Disable RTN-mode optimization (iters=0). Defaults to False.enable_alg_ext (bool, optional): Enable algorithm extension (primarily for INT2). Defaults to False.+ model_dtype (str): model dtype used to load pre-trained model.**kwargs: Backward compatible options:- enable_alg_ext, quant_lm_head, lr, lr_scheduler, sampler, not_use_best_mse, dynamic_max_gap,super_group_size, super_bits, scale_dtype ("fp16" etc.),diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex c5b500ce1..5e00f92a4 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -230,6 +230,7 @@ class BaseCompressor(object):disable_deterministic_algorithms = kwargs.pop("disable_deterministic_algorithms", True)enable_deterministic_algorithms = kwargs.pop("enable_deterministic_algorithms", False)static_kv_dtype = kwargs.pop("static_kv_dtype", None)+ model_dtype = kwargs.pop("model_dtype", None)device = kwargs.pop("device", None)if envs.AR_USE_MODELSCOPE:platform = "model_scope"@@ -267,6 +268,7 @@ class BaseCompressor(object):model,platform=platform,device="cpu", # always load cpu first+ model_dtype=model_dtype,)elif tokenizer is None and not self.diffusion and iters > 0:raise ValueError("A tokenizer must be set for non-str model input")@@ -490,6 +492,7 @@ class BaseCompressor(object):# We’d better keep the string scheme instead of the dict config,# since GGUF uses different mixed-bit strategies for q4_k_s and q4_k_m# even though they share the same scheme dict.+ scheme = scheme.strip("'\" ")res = schemescheme = scheme.upper()scheme = asdict(preset_name_to_scheme(scheme))diff --git a/auto_round/compressors/diffusion/compressor.py b/auto_round/compressors/diffusion/compressor.pyindex 904767602..f8680774f 100644--- a/auto_round/compressors/diffusion/compressor.py+++ b/auto_round/compressors/diffusion/compressor.py@@ -101,6 +101,7 @@ class DiffusionCompressor(BaseCompressor):**kwargs,):logger.warning("Diffusion model quantization is experimental and is only validated on Flux models.")+ model_dtype = kwargs.pop("model_dtype", None)self.guidance_scale = guidance_scaleself.num_inference_steps = num_inference_steps@@ -112,7 +113,7 @@ class DiffusionCompressor(BaseCompressor):self._set_device(device_map)if isinstance(model, str):- pipe, model = diffusion_load_model(model, platform=platform, device=self.device)+ pipe, model = diffusion_load_model(model, platform=platform, device=self.device, model_dtype=model_dtype)elif isinstance(model, pipeline_utils.DiffusionPipeline):pipe = modelmodel = pipe.transformerdiff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.pyindex 94210388f..6379ac341 100644--- a/auto_round/compressors/mllm/compressor.py+++ b/auto_round/compressors/mllm/compressor.py@@ -168,6 +168,7 @@ class MLLMCompressor(BaseCompressor):extra_data_dir = kwargs.pop("extra_data_dir", None)template = kwargs.pop("template", None)+ model_dtype = kwargs.pop("model_dtype", None)to_quant_block_names: Union[str, list, None] = kwargs.pop("to_quant_block_names", None)if device_map is None:@@ -175,7 +176,9 @@ class MLLMCompressor(BaseCompressor):self._set_device(device_map)if isinstance(model, str):- model, processor, tokenizer, image_processor = mllm_load_model(model, platform=platform, device=self.device)+ model, processor, tokenizer, image_processor = mllm_load_model(+ model, platform=platform, device=self.device, model_dtype=model_dtype+ )self.model = modelquant_nontext_module = self._check_quant_nontext(layer_config, quant_nontext_module)diff --git a/auto_round/eval/eval_cli.py b/auto_round/eval/eval_cli.pyindex 4311cdeb8..009b6458d 100644--- a/auto_round/eval/eval_cli.py+++ b/auto_round/eval/eval_cli.py@@ -220,6 +220,7 @@ def eval_task_by_task(trust_remote_code=True,eval_model_dtype=None,retry_times=3,+ mllm=False,):set_cuda_visible_devices(device)device_str, parallelism = get_device_and_parallelism(device)@@ -228,6 +229,7 @@ def eval_task_by_task(import tracebackfrom lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611+ from lm_eval.models.hf_vlms import HFMultimodalLMfrom lm_eval.models.huggingface import HFLMfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -263,16 +265,31 @@ def eval_task_by_task()model.eval()parallelism = False- hflm = HFLM(- pretrained=model,- tokenizer=tokenizer,- device=device_str,- batch_size=batch_size,- max_batch_size=max_batch_size,- parallelize=parallelism,- trust_remote_code=trust_remote_code,- dtype=eval_model_dtype,- )+ if mllm:+ if batch_size is None or batch_size == "auto":+ logger.warning("hf-multimodal models does not support auto currently, reset eval_bs to 16")+ batch_size = 16+ hflm = HFMultimodalLM(+ pretrained=model,+ tokenizer=tokenizer,+ device=device_str,+ batch_size=batch_size,+ max_batch_size=max_batch_size,+ parallelize=parallelism,+ trust_remote_code=trust_remote_code,+ dtype=eval_model_dtype,+ )+ else:+ hflm = HFLM(+ pretrained=model,+ tokenizer=tokenizer,+ device=device_str,+ batch_size=batch_size,+ max_batch_size=max_batch_size,+ parallelize=parallelism,+ trust_remote_code=trust_remote_code,+ dtype=eval_model_dtype,+ )if isinstance(tasks, str):tasks = tasks.replace(" ", "").split(",")diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex 9722b6696..00a0fdca0 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -17,8 +17,11 @@ from typing import Optional, Unionfrom lm_eval import simple_evaluate as lm_simple_evaluate # pylint: disable=E0611+from auto_round.logger import logger+os.environ["TOKENIZERS_PARALLELISM"] = "false"+from lm_eval.models.hf_vlms import HFMultimodalLMfrom lm_eval.models.huggingface import HFLM@@ -30,16 +33,30 @@ def simple_evaluate_user_model(max_batch_size: Optional[int] = 64,eval_model_dtype="auto",add_bos_token: bool = False,+ mllm: bool = False,**kwargs):- hflm = HFLM(- pretrained=user_model,- tokenizer=tokenizer,- batch_size=batch_size,- max_batch_size=max_batch_size,- dtype=eval_model_dtype,- add_bos_token=add_bos_token,- )+ if mllm:+ if batch_size is None or batch_size == "auto":+ logger.warning("hf-multimodal models does not support auto currently, reset eval_bs to 16")+ batch_size = 16+ hflm = HFMultimodalLM(+ pretrained=user_model,+ tokenizer=tokenizer,+ batch_size=batch_size,+ max_batch_size=max_batch_size,+ dtype=eval_model_dtype,+ add_bos_token=add_bos_token,+ )+ else:+ hflm = HFLM(+ pretrained=user_model,+ tokenizer=tokenizer,+ batch_size=batch_size,+ max_batch_size=max_batch_size,+ dtype=eval_model_dtype,+ add_bos_token=add_bos_token,+ )return lm_simple_evaluate(model=hflm, model_args=None, batch_size=batch_size, max_batch_size=max_batch_size, limit=limit, **kwargs)