PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
update autoround mllm and support Mistral 3.2 series (#713)
Heng Guo2e911b762547
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 8c682ec3a..a85e99a3f 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -15,24 +15,31 @@ import sysdef run_eval():- from auto_round.script.llm import setup_eval_parser-- args = setup_eval_parser()- if args.eval_task_by_task:- from auto_round.script.llm import eval_task_by_task-- eval_task_by_task(- model=args.model,- device=args.device,- tasks=args.tasks,- batch_size=args.eval_bs,- trust_remote_code=not args.disable_trust_remote_code,- eval_model_dtype=args.eval_model_dtype,- )+ if "--vlmeval" in sys.argv:+ sys.argv.remove("--vlmeval")+ run_vlmeavl()+ elif "--lmms" in sys.argv:+ sys.argv.remove("--lmms")+ run_lmms()else:- from auto_round.script.llm import eval+ from auto_round.script.llm import setup_eval_parser++ args = setup_eval_parser()+ if args.eval_task_by_task:+ from auto_round.script.llm import eval_task_by_task- eval(args)+ eval_task_by_task(+ model=args.model,+ device=args.device,+ tasks=args.tasks,+ batch_size=args.eval_bs,+ trust_remote_code=not args.disable_trust_remote_code,+ eval_model_dtype=args.eval_model_dtype,+ )+ else:+ from auto_round.script.llm import eval++ eval(args)def run():@@ -67,27 +74,6 @@ def run_fast():tune(args)-def run_mllm():- if "--eval" in sys.argv:- from auto_round.script.llm import eval, setup_eval_parser-- sys.argv.remove("--eval")- args = setup_eval_parser()- args.mllm = True- eval(args)- elif "--vlmeval" in sys.argv:- sys.argv.remove("--vlmeval")- run_vlmeavl()- elif "--lmms" in sys.argv:- sys.argv.remove("--lmms")- run_lmms()- else:- from auto_round.script.mllm import setup_parser, tune-- args = setup_parser()- tune(args)--def run_lmms():# from auto_round.script.lmms_eval import setup_lmms_args, evalfrom auto_round.script.mllm import lmms_eval, setup_lmms_parser@@ -103,13 +89,5 @@ def run_vlmeavl():vlmeval(args)-def switch():- if "--mllm" in sys.argv:- sys.argv.remove("--mllm")- run_mllm()- else:- run()--if __name__ == "__main__":- switch()+ run()diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 10d312ab4..cd49637a8 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -138,7 +138,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllimage_processor = kwargs.get("image_processor", None)if output_dir is not None and os.path.exists(output_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")- if output_dir is not None and tokenizer is not None:+ if output_dir is not None and tokenizer is not None and hasattr(tokenizer, "save_pretrained"):tokenizer.save_pretrained(output_dir)if output_dir is not None and processor is not None:processor.save_pretrained(output_dir)@@ -240,7 +240,12 @@ def save("""##max_shard_size = "10000GB" ## API of auto-gptq with marlin does not support shard sizeos.makedirs(save_dir, exist_ok=True)- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ try:+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ except ValueError as e:+ if hasattr(model, "generation_config"):+ setattr(model.generation_config, "do_sample", True)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):with open(config_path, "r") as file:diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 89d25d9cb..3e72c9590 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -350,7 +350,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exreturn modelif os.path.exists(output_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")- if tokenizer is not None:+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):tokenizer.save_pretrained(output_dir)if processor is not None:@@ -389,7 +389,13 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriWhether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`)."""os.makedirs(save_dir, exist_ok=True)- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ try:+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ except ValueError as e:+ if hasattr(model, "generation_config"):+ setattr(model.generation_config, "do_sample", True)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):with open(config_path, "r") as file:diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex 03a92a922..760a7205e 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -93,7 +93,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):logger.warning(f"{output_dir} already exists, this may cause model conflict")logger.info("Saving quantized model to auto_awq format")- if tokenizer is not None and output_dir is not None:+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained") and output_dir is not None:tokenizer.save_pretrained(output_dir)if processor is not None and output_dir is not None:processor.save_pretrained(output_dir)@@ -178,7 +178,12 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seriWhether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`)."""os.makedirs(save_dir, exist_ok=True)- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ try:+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)+ except ValueError as e:+ if hasattr(model, "generation_config"):+ setattr(model.generation_config, "do_sample", True)+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)config_path = os.path.join(save_dir, "config.json")if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):with open(config_path, "r") as file:diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 31d2fcbd2..9074deb0a 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -129,7 +129,7 @@ def pack_gguf_layer(if "gguf_model_instance_global" not in globals():config = model.configconfig.save_pretrained(tmp_work_dir)- if tokenizer is not None:+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):tokenizer.save_pretrained(tmp_work_dir)if processor is not None:processor.save_pretrained(tmp_work_dir)diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex bc233eebe..731c93d2f 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -98,7 +98,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):quantize_config.save_pretrained(output_dir)try:compressed_model.save_pretrained(output_dir, safe_serialization=safe_serialization)- if tokenizer is not None:+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):tokenizer.save_pretrained(output_dir)logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))except IOError as e: # pragma: no coverdiff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.pyindex 161a44ed1..e5863f374 100644--- a/auto_round/export/export_to_llmcompressor/export.py+++ b/auto_round/export/export_to_llmcompressor/export.py@@ -45,7 +45,7 @@ def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):logger.warning(f"{output_dir} already exists, this may cause model conflict")# save tokenizer, processor- if output_dir is not None and tokenizer is not None:+ if output_dir is not None and tokenizer is not None and hasattr(tokenizer, "save_pretrained"):tokenizer.save_pretrained(output_dir)if output_dir is not None and processor is not None:processor.save_pretrained(output_dir)@@ -71,4 +71,9 @@ def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):# save model.config, model.state_dict()model.config.quantization_config = quantization_configmodel.config.save_pretrained(output_dir)- model.save_pretrained(output_dir, safe_serialization=safe_serialization)+ try:+ model.save_pretrained(output_dir, safe_serialization=safe_serialization)+ except ValueError as e:+ if hasattr(model, "generation_config"):+ setattr(model.generation_config, "do_sample", True)+ model.save_pretrained(output_dir, safe_serialization=safe_serialization)diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 4aaa562f5..95ba736bd 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -18,7 +18,11 @@ from typing import Unionimport torchfrom tqdm import tqdm-from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS, _handle_special_model+from auto_round.special_model_handler import (+ NOT_SUPPORT_ONLY_TEXT_MODELS,+ SUPPORT_ONLY_TEXT_MODELS,+ _handle_special_model,+)from ..autoround import AutoRoundfrom ..low_cpu_mem.utils import get_layers_before_block@@ -43,11 +47,14 @@ def _only_text_test(model, tokenizer, device, model_type):if model_type in SUPPORT_ONLY_TEXT_MODELS: # save timereturn True+ if model_type in NOT_SUPPORT_ONLY_TEXT_MODELS:+ return False+new_tokenizer = deepcopy(tokenizer)device = detect_device(device)text = ["only text", "test"]new_tokenizer.padding_side = "left"- if new_tokenizer.pad_token is None:+ if hasattr(new_tokenizer, "pad_token") and new_tokenizer.pad_token is None:new_tokenizer.pad_token = new_tokenizer.eos_tokeninputs = new_tokenizer(text, return_tensors="pt", padding=True, truncation=True)@@ -179,25 +186,30 @@ class AutoRoundMLLM(AutoRound):if model.config.model_type == "llava" and isinstance(model, PreTrainedModel):template = "default"- self.template = template if template is not None else model.config.model_type- if not isinstance(dataset, torch.utils.data.DataLoader):- self.template = get_template(- self.template,- model=model,- tokenizer=tokenizer,- processor=processor,- image_processor=image_processor,- use_rtn=iters == 0,- quiet=not self.quant_nontext_module,- )- dataset = self.template.default_dataset if dataset is None else dataset+ if hasattr(model, "name_or_path") and "Mistral-Small-3.2" in model.name_or_path:+ template = "mistral3_2"+ if iters > 0:+ self.template = template if template is not None else model.config.model_type+ if not isinstance(dataset, torch.utils.data.DataLoader):+ self.template = get_template(+ self.template,+ model=model,+ tokenizer=tokenizer,+ processor=processor,+ image_processor=image_processor,+ use_rtn=iters == 0,+ quiet=not self.quant_nontext_module,+ )+ dataset = self.template.default_dataset if dataset is None else dataset+ else:+ self.template = Nonemodel = _handle_special_model(model)from ..calib_dataset import CALIB_DATASETSfrom .mllm_dataset import MLLM_DATASET- if isinstance(dataset, str) and dataset in CALIB_DATASETS.keys():+ if iters > 0 and isinstance(dataset, str) and dataset in CALIB_DATASETS.keys():if quant_nontext_module:logger.warning("Text only dataset cannot be used for calibrating non-text modules,"diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.pyindex ac021e13b..62e76e488 100644--- a/auto_round/mllm/processor.py+++ b/auto_round/mllm/processor.py@@ -27,6 +27,9 @@ Support Matrix✔ means support, - means support but cannot infer or not test infert yet, X means not support."""+import os+from datetime import datetime, timedelta+import torchfrom transformers.data.data_collator import default_data_collator@@ -334,3 +337,71 @@ class DeepSeekV2Processor(BasicProcessor):prepare_inputs = prepare_inputs.to(self.model.device)prepare_inputs = self.squeeze_result(dict(prepare_inputs))return prepare_inputs+++@register_processor("mistral3_2")+class Mistral3Processor(BasicProcessor):+ IMAGE_TOKEN = "<image>"++ @staticmethod+ def load_system_prompt(repo_id_or_path: str, filename: str) -> str:+ from huggingface_hub import hf_hub_download++ if os.path.isdir(repo_id_or_path):+ file_path = os.path.join(repo_id_or_path, filename)+ else:+ file_path = hf_hub_download(repo_id=repo_id_or_path, filename=filename)+ with open(file_path, "r") as file:+ system_prompt = file.read()+ today = datetime.today().strftime("%Y-%m-%d")+ yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y-%m-%d")+ model_name = repo_id_or_path.split("/")[-1]+ return system_prompt.format(name=model_name, today=today, yesterday=yesterday)++ def get_input(+ self,+ text,+ images,+ return_tensors="pt",+ squeeze=True,+ max_length=None,+ truncation=False,+ truncation_strategy="text",+ **kwargs+ ):+ from mistral_common.protocol.instruct.request import ChatCompletionRequest # pylint: disable=E0401++ SYSTEM_PROMPT = self.load_system_prompt(self.model.name_or_path, "SYSTEM_PROMPT.txt")++ conversation = [{"role": "system", "content": SYSTEM_PROMPT}]+ for content in text:+ conversation.append(+ {+ "role": content["role"],+ "content": [+ {+ "type": "text",+ "text": content["content"].replace(self.IMAGE_TOKEN, ""),+ },+ ],+ },+ )+ if self.IMAGE_TOKEN in content["content"]:+ conversation[-1]["content"].append({"type": "image_url", "image_url": {"url": images}})+ tokenized = self.tokenizer.encode_chat_completion(+ ChatCompletionRequest(messages=conversation, continue_final_message=True)+ )+ input_ids = torch.tensor([tokenized.tokens])+ attention_mask = torch.ones_like(input_ids)+ pixel_values = torch.tensor(tokenized.images[0], dtype=torch.bfloat16).unsqueeze(0)+ image_sizes = torch.tensor([pixel_values.shape[-2:]])++ ret = {+ "input_ids": input_ids,+ "attention_mask": attention_mask,+ "pixel_values": pixel_values,+ "image_sizes": image_sizes,+ }+ if squeeze:+ ret = self.squeeze_result(ret)+ return retdiff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex ec74d64e5..7f4531e07 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -121,6 +121,7 @@ _register_template("qwen2_5_vl", default_dataset="NeelNanda/pile-10k", processor_register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCESSORS["hf"])_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["deepseek_v2"])_register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])+_register_template("mistral3_2", default_dataset="liuhaotian/llava", processor=PROCESSORS["mistral3_2"])def load_template(path: str):diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 0245a7f72..5e10a5590 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -53,6 +53,8 @@ class BasicArgumentParser(argparse.ArgumentParser):"--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path")+ self.add_argument("--mllm", action="store_true", help="whether to quant multi-modal model.")+self.add_argument("--eval", action="store_true", help="whether to use eval only mode")self.add_argument("--bits", default=4, type=int, help="number of weight bits")@@ -226,6 +228,31 @@ class BasicArgumentParser(argparse.ArgumentParser):help="whether to disable optimization of the RTN mode(iters=0) (default is False).",)+ ## ======================= MLLM =======================+ self.add_argument(+ "--quant_nontext_module",+ action="store_true",+ help="whether to quantize non-text module, e.g. vision component",+ )++ self.add_argument(+ "--extra_data_dir",+ default=None,+ type=str,+ help="dataset dir for storing images/audio/videos. "+ "Can be a dir path or multiple dir path with format as "+ "'image=path_to_image,video=path_to_video,audio=path_to_audio'"+ "By default, it will search in the relative path, "+ "and if not find, will automatic download.",+ )++ self.add_argument(+ "--template",+ default=None,+ type=str,+ help="the template for building training dataset. It can be a custom one.",+ )+class EvalArgumentParser(argparse.ArgumentParser):@@ -234,7 +261,7 @@ class EvalArgumentParser(argparse.ArgumentParser):self.add_argument("--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path")- self.add_argument("--mllm", default=False, help="whether to eval multi-modal model.")+ self.add_argument("--mllm", action="store_true", help="whether to eval multi-modal model.")self.add_argument("--device","--devices",@@ -419,23 +446,46 @@ def tune(args):model_name = model_name[:-1]logger.info(f"start to quantize {model_name}")- from auto_round.utils import llm_load_model+ from auto_round.utils import llm_load_model, mllm_load_model- model, tokenizer, low_cpu_mem_usage = llm_load_model(- model_name,- trust_remote_code=not args.disable_trust_remote_code,- device=args.device,- low_cpu_mem_mode=args.low_cpu_mem_mode,- low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,- model_dtype=args.model_dtype,- )+ if args.mllm:+ model, processor, tokenizer, image_processor = mllm_load_model(+ model_name,+ device=device_str,+ use_auto_mapping=use_auto_mapping,+ trust_remote_code=not args.disable_trust_remote_code,+ model_dtype=args.model_dtype,+ )+ low_cpu_mem_usage = False+ else:+ model, tokenizer, low_cpu_mem_usage = llm_load_model(+ model_name,+ use_auto_mapping=use_auto_mapping,+ trust_remote_code=not args.disable_trust_remote_code,+ device=device_str,+ low_cpu_mem_mode=args.low_cpu_mem_mode,+ low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,+ model_dtype=args.model_dtype,+ )- from auto_round import AutoRound, AutoRoundAdam+ from auto_round import AutoRound, AutoRoundAdam, AutoRoundMLLMif "bloom" in model_name:args.low_gpu_mem_usage = False- round = AutoRound if not args.adam else AutoRoundAdam+ round = AutoRound+ mllm_kwargs = {}+ if args.adam:+ round = AutoRoundAdam+ if args.mllm:+ round = AutoRoundMLLM+ mllm_kwargs = {+ "processor": processor,+ "image_processor": image_processor,+ "extra_data_dir": args.extra_data_dir,+ "quant_nontext_module": args.quant_nontext_module,+ "template": args.template,+ }layer_config = {}not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)@@ -485,10 +535,10 @@ def tune(args):enable_torch_compile = True if "--enable_torch_compile" in sys.argv else Falseautoround = round(- model,- tokenizer,- args.bits,- args.group_size,+ model=model,+ tokenizer=tokenizer,+ bits=args.bits,+ group_size=args.group_size,sym=not args.asym,batch_size=args.batch_size,dataset=args.dataset,@@ -521,6 +571,7 @@ def tune(args):super_group_size=args.super_group_size,super_bits=args.super_bits,disable_opt_rtn=args.disable_opt_rtn,+ **mllm_kwargs,)model_name = args.model.rstrip("/")diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex c381854a9..9c3933610 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -28,6 +28,8 @@ SUPPORT_ONLY_TEXT_MODELS = ["internvl_chat",]+NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]+SPECIAL_SHARED_CACHE_KEYS = {"Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")}diff --git a/auto_round/utils.py b/auto_round/utils.pyindex dc738c122..0c00754e4 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1379,6 +1379,7 @@ def llm_load_model(def mllm_load_model(pretrained_model_name_or_path,+ device="cpu",torch_dtype="auto",use_auto_mapping=True,trust_remote_code=True,@@ -1391,6 +1392,10 @@ def mllm_load_model(from huggingface_hub import HfApi, HfFileSystem, hf_hub_downloadfrom transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer+ device_str, use_auto_mapping = get_device_and_parallelism(device)+ torch_dtype = "auto"+ if device_str is not None and "hpu" in device_str:+ torch_dtype = torch.bfloat16if os.path.isdir(pretrained_model_name_or_path):config = json.load(open(os.path.join(pretrained_model_name_or_path, "config.json")))else:@@ -1455,12 +1460,21 @@ def mllm_load_model(torch_dtype=torch_dtype,device_map="auto" if use_auto_mapping else None,)- tokenizer = AutoTokenizer.from_pretrained(- pretrained_model_name_or_path, trust_remote_code=trust_remote_code- )- processor = AutoProcessor.from_pretrained(- pretrained_model_name_or_path, trust_remote_code=trust_remote_code- )++ if "Mistral-Small-3.2" in pretrained_model_name_or_path:+ from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # pylint: disable=E0401++ if os.path.isdir(pretrained_model_name_or_path):+ tokenizer = MistralTokenizer.from_file(os.path.join(pretrained_model_name_or_path, "tekken.json"))+ else:+ tokenizer = MistralTokenizer.from_hf_hub(pretrained_model_name_or_path)+ else:+ tokenizer = AutoTokenizer.from_pretrained(+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code+ )+ processor = AutoProcessor.from_pretrained(+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code+ )try:from transformers import AutoImageProcessordiff --git a/test/test_cpu/test_basic_usage.py b/test/test_cpu/test_basic_usage.pyindex 7de1204c2..93266c2b7 100644--- a/test/test_cpu/test_basic_usage.py+++ b/test/test_cpu/test_basic_usage.py@@ -44,29 +44,25 @@ class TestAutoRoundCmd(unittest.TestCase):assert False, "cmd line test fail, please have a check"# test mllm script- # test auto_round_mllm help- res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm -h")- if res > 0 or res == -1:- assert False, "cmd line test fail, please have a check"# test auto_round_mllm --eval help- res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm --eval -h")+ res = os.system(f"cd ../.. && {python_path} -m auto_round --eval -h")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"# test auto_round_mllm --lmms help- res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm --lmms -h")+ res = os.system(f"cd ../.. && {python_path} -m auto_round --eval --lmms -h")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"res = os.system(- f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved"+ f"cd ../.. && {python_path} -m auto_round --mllm --model Qwen/Qwen2-VL-2B-Instruct --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved")if res > 0 or res == -1:assert False, "cmd line test fail, please have a check"res = os.system(- f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"+ f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --model Qwen/Qwen2-VL-2B-Instruct --seqlen 256 --format auto_round"" --quant_nontext_module --output_dir ./saved ")if res > 0 or res == -1: