Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

update autoround mllm and support Mistral 3.2 series (#713)

Heng Guo2e911b762547
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index 8c682ec3a..a85e99a3f 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -15,24 +15,31 @@ import sys
def run_eval():
- from auto_round.script.llm import setup_eval_parser
-
- args = setup_eval_parser()
- if args.eval_task_by_task:
- from auto_round.script.llm import eval_task_by_task
-
- eval_task_by_task(
- model=args.model,
- device=args.device,
- tasks=args.tasks,
- batch_size=args.eval_bs,
- trust_remote_code=not args.disable_trust_remote_code,
- eval_model_dtype=args.eval_model_dtype,
- )
+ if "--vlmeval" in sys.argv:
+ sys.argv.remove("--vlmeval")
+ run_vlmeavl()
+ elif "--lmms" in sys.argv:
+ sys.argv.remove("--lmms")
+ run_lmms()
else:
- from auto_round.script.llm import eval
+ from auto_round.script.llm import setup_eval_parser
+
+ args = setup_eval_parser()
+ if args.eval_task_by_task:
+ from auto_round.script.llm import eval_task_by_task
- eval(args)
+ eval_task_by_task(
+ model=args.model,
+ device=args.device,
+ tasks=args.tasks,
+ batch_size=args.eval_bs,
+ trust_remote_code=not args.disable_trust_remote_code,
+ eval_model_dtype=args.eval_model_dtype,
+ )
+ else:
+ from auto_round.script.llm import eval
+
+ eval(args)
def run():
@@ -67,27 +74,6 @@ def run_fast():
tune(args)
-def run_mllm():
- if "--eval" in sys.argv:
- from auto_round.script.llm import eval, setup_eval_parser
-
- sys.argv.remove("--eval")
- args = setup_eval_parser()
- args.mllm = True
- eval(args)
- elif "--vlmeval" in sys.argv:
- sys.argv.remove("--vlmeval")
- run_vlmeavl()
- elif "--lmms" in sys.argv:
- sys.argv.remove("--lmms")
- run_lmms()
- else:
- from auto_round.script.mllm import setup_parser, tune
-
- args = setup_parser()
- tune(args)
-
-
def run_lmms():
# from auto_round.script.lmms_eval import setup_lmms_args, eval
from auto_round.script.mllm import lmms_eval, setup_lmms_parser
@@ -103,13 +89,5 @@ def run_vlmeavl():
vlmeval(args)
-def switch():
- if "--mllm" in sys.argv:
- sys.argv.remove("--mllm")
- run_mllm()
- else:
- run()
-
-
if __name__ == "__main__":
- switch()
+ run()
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 10d312ab4..cd49637a8 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -138,7 +138,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
image_processor = kwargs.get("image_processor", None)
if output_dir is not None and os.path.exists(output_dir):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
- if output_dir is not None and tokenizer is not None:
+ if output_dir is not None and tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
tokenizer.save_pretrained(output_dir)
if output_dir is not None and processor is not None:
processor.save_pretrained(output_dir)
@@ -240,7 +240,12 @@ def save(
"""
##max_shard_size = "10000GB" ## API of auto-gptq with marlin does not support shard size
os.makedirs(save_dir, exist_ok=True)
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ try:
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ except ValueError as e:
+ if hasattr(model, "generation_config"):
+ setattr(model.generation_config, "do_sample", True)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
config_path = os.path.join(save_dir, "config.json")
if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
with open(config_path, "r") as file:
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 89d25d9cb..3e72c9590 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -350,7 +350,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
return model
if os.path.exists(output_dir):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
- if tokenizer is not None:
+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
tokenizer.save_pretrained(output_dir)
if processor is not None:
@@ -389,7 +389,13 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
"""
os.makedirs(save_dir, exist_ok=True)
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ try:
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ except ValueError as e:
+ if hasattr(model, "generation_config"):
+ setattr(model.generation_config, "do_sample", True)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+
config_path = os.path.join(save_dir, "config.json")
if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
with open(config_path, "r") as file:
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index 03a92a922..760a7205e 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -93,7 +93,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
logger.info("Saving quantized model to auto_awq format")
- if tokenizer is not None and output_dir is not None:
+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained") and output_dir is not None:
tokenizer.save_pretrained(output_dir)
if processor is not None and output_dir is not None:
processor.save_pretrained(output_dir)
@@ -178,7 +178,12 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "5GB", safe_seri
Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
"""
os.makedirs(save_dir, exist_ok=True)
- model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ try:
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
+ except ValueError as e:
+ if hasattr(model, "generation_config"):
+ setattr(model.generation_config, "do_sample", True)
+ model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
config_path = os.path.join(save_dir, "config.json")
if dtype is not None and dtype != model.dtype and os.path.exists(os.path.join(save_dir, "config.json")):
with open(config_path, "r") as file:
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index 31d2fcbd2..9074deb0a 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -129,7 +129,7 @@ def pack_gguf_layer(
if "gguf_model_instance_global" not in globals():
config = model.config
config.save_pretrained(tmp_work_dir)
- if tokenizer is not None:
+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
tokenizer.save_pretrained(tmp_work_dir)
if processor is not None:
processor.save_pretrained(tmp_work_dir)
diff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.py
index bc233eebe..731c93d2f 100644
--- a/auto_round/export/export_to_itrex/export.py
+++ b/auto_round/export/export_to_itrex/export.py
@@ -98,7 +98,7 @@ def save_quantized_as_itrex(output_dir, inplace=True, **kwargs):
quantize_config.save_pretrained(output_dir)
try:
compressed_model.save_pretrained(output_dir, safe_serialization=safe_serialization)
- if tokenizer is not None:
+ if tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
tokenizer.save_pretrained(output_dir)
logger.info("Saved config file and weights of quantized model to {}.".format(output_dir))
except IOError as e: # pragma: no cover
diff --git a/auto_round/export/export_to_llmcompressor/export.py b/auto_round/export/export_to_llmcompressor/export.py
index 161a44ed1..e5863f374 100644
--- a/auto_round/export/export_to_llmcompressor/export.py
+++ b/auto_round/export/export_to_llmcompressor/export.py
@@ -45,7 +45,7 @@ def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
# save tokenizer, processor
- if output_dir is not None and tokenizer is not None:
+ if output_dir is not None and tokenizer is not None and hasattr(tokenizer, "save_pretrained"):
tokenizer.save_pretrained(output_dir)
if output_dir is not None and processor is not None:
processor.save_pretrained(output_dir)
@@ -71,4 +71,9 @@ def save_quantized_as_llmcompressor(output_dir, model=None, **kwargs):
# save model.config, model.state_dict()
model.config.quantization_config = quantization_config
model.config.save_pretrained(output_dir)
- model.save_pretrained(output_dir, safe_serialization=safe_serialization)
+ try:
+ model.save_pretrained(output_dir, safe_serialization=safe_serialization)
+ except ValueError as e:
+ if hasattr(model, "generation_config"):
+ setattr(model.generation_config, "do_sample", True)
+ model.save_pretrained(output_dir, safe_serialization=safe_serialization)
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 4aaa562f5..95ba736bd 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -18,7 +18,11 @@ from typing import Union
import torch
from tqdm import tqdm
-from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS, _handle_special_model
+from auto_round.special_model_handler import (
+ NOT_SUPPORT_ONLY_TEXT_MODELS,
+ SUPPORT_ONLY_TEXT_MODELS,
+ _handle_special_model,
+)
from ..autoround import AutoRound
from ..low_cpu_mem.utils import get_layers_before_block
@@ -43,11 +47,14 @@ def _only_text_test(model, tokenizer, device, model_type):
if model_type in SUPPORT_ONLY_TEXT_MODELS: # save time
return True
+ if model_type in NOT_SUPPORT_ONLY_TEXT_MODELS:
+ return False
+
new_tokenizer = deepcopy(tokenizer)
device = detect_device(device)
text = ["only text", "test"]
new_tokenizer.padding_side = "left"
- if new_tokenizer.pad_token is None:
+ if hasattr(new_tokenizer, "pad_token") and new_tokenizer.pad_token is None:
new_tokenizer.pad_token = new_tokenizer.eos_token
inputs = new_tokenizer(text, return_tensors="pt", padding=True, truncation=True)
@@ -179,25 +186,30 @@ class AutoRoundMLLM(AutoRound):
if model.config.model_type == "llava" and isinstance(model, PreTrainedModel):
template = "default"
- self.template = template if template is not None else model.config.model_type
- if not isinstance(dataset, torch.utils.data.DataLoader):
- self.template = get_template(
- self.template,
- model=model,
- tokenizer=tokenizer,
- processor=processor,
- image_processor=image_processor,
- use_rtn=iters == 0,
- quiet=not self.quant_nontext_module,
- )
- dataset = self.template.default_dataset if dataset is None else dataset
+ if hasattr(model, "name_or_path") and "Mistral-Small-3.2" in model.name_or_path:
+ template = "mistral3_2"
+ if iters > 0:
+ self.template = template if template is not None else model.config.model_type
+ if not isinstance(dataset, torch.utils.data.DataLoader):
+ self.template = get_template(
+ self.template,
+ model=model,
+ tokenizer=tokenizer,
+ processor=processor,
+ image_processor=image_processor,
+ use_rtn=iters == 0,
+ quiet=not self.quant_nontext_module,
+ )
+ dataset = self.template.default_dataset if dataset is None else dataset
+ else:
+ self.template = None
model = _handle_special_model(model)
from ..calib_dataset import CALIB_DATASETS
from .mllm_dataset import MLLM_DATASET
- if isinstance(dataset, str) and dataset in CALIB_DATASETS.keys():
+ if iters > 0 and isinstance(dataset, str) and dataset in CALIB_DATASETS.keys():
if quant_nontext_module:
logger.warning(
"Text only dataset cannot be used for calibrating non-text modules,"
diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.py
index ac021e13b..62e76e488 100644
--- a/auto_round/mllm/processor.py
+++ b/auto_round/mllm/processor.py
@@ -27,6 +27,9 @@ Support Matrix
✔ means support, - means support but cannot infer or not test infert yet, X means not support.
"""
+import os
+from datetime import datetime, timedelta
+
import torch
from transformers.data.data_collator import default_data_collator
@@ -334,3 +337,71 @@ class DeepSeekV2Processor(BasicProcessor):
prepare_inputs = prepare_inputs.to(self.model.device)
prepare_inputs = self.squeeze_result(dict(prepare_inputs))
return prepare_inputs
+
+
+@register_processor("mistral3_2")
+class Mistral3Processor(BasicProcessor):
+ IMAGE_TOKEN = "<image>"
+
+ @staticmethod
+ def load_system_prompt(repo_id_or_path: str, filename: str) -> str:
+ from huggingface_hub import hf_hub_download
+
+ if os.path.isdir(repo_id_or_path):
+ file_path = os.path.join(repo_id_or_path, filename)
+ else:
+ file_path = hf_hub_download(repo_id=repo_id_or_path, filename=filename)
+ with open(file_path, "r") as file:
+ system_prompt = file.read()
+ today = datetime.today().strftime("%Y-%m-%d")
+ yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y-%m-%d")
+ model_name = repo_id_or_path.split("/")[-1]
+ return system_prompt.format(name=model_name, today=today, yesterday=yesterday)
+
+ def get_input(
+ self,
+ text,
+ images,
+ return_tensors="pt",
+ squeeze=True,
+ max_length=None,
+ truncation=False,
+ truncation_strategy="text",
+ **kwargs
+ ):
+ from mistral_common.protocol.instruct.request import ChatCompletionRequest # pylint: disable=E0401
+
+ SYSTEM_PROMPT = self.load_system_prompt(self.model.name_or_path, "SYSTEM_PROMPT.txt")
+
+ conversation = [{"role": "system", "content": SYSTEM_PROMPT}]
+ for content in text:
+ conversation.append(
+ {
+ "role": content["role"],
+ "content": [
+ {
+ "type": "text",
+ "text": content["content"].replace(self.IMAGE_TOKEN, ""),
+ },
+ ],
+ },
+ )
+ if self.IMAGE_TOKEN in content["content"]:
+ conversation[-1]["content"].append({"type": "image_url", "image_url": {"url": images}})
+ tokenized = self.tokenizer.encode_chat_completion(
+ ChatCompletionRequest(messages=conversation, continue_final_message=True)
+ )
+ input_ids = torch.tensor([tokenized.tokens])
+ attention_mask = torch.ones_like(input_ids)
+ pixel_values = torch.tensor(tokenized.images[0], dtype=torch.bfloat16).unsqueeze(0)
+ image_sizes = torch.tensor([pixel_values.shape[-2:]])
+
+ ret = {
+ "input_ids": input_ids,
+ "attention_mask": attention_mask,
+ "pixel_values": pixel_values,
+ "image_sizes": image_sizes,
+ }
+ if squeeze:
+ ret = self.squeeze_result(ret)
+ return ret
diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.py
index ec74d64e5..7f4531e07 100644
--- a/auto_round/mllm/template.py
+++ b/auto_round/mllm/template.py
@@ -121,6 +121,7 @@ _register_template("qwen2_5_vl", default_dataset="NeelNanda/pile-10k", processor
_register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCESSORS["hf"])
_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["deepseek_v2"])
_register_template("mistral3", default_dataset="NeelNanda/pile-10k", processor=PROCESSORS["hf"])
+_register_template("mistral3_2", default_dataset="liuhaotian/llava", processor=PROCESSORS["mistral3_2"])
def load_template(path: str):
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 0245a7f72..5e10a5590 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -53,6 +53,8 @@ class BasicArgumentParser(argparse.ArgumentParser):
"--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"
)
+ self.add_argument("--mllm", action="store_true", help="whether to quant multi-modal model.")
+
self.add_argument("--eval", action="store_true", help="whether to use eval only mode")
self.add_argument("--bits", default=4, type=int, help="number of weight bits")
@@ -226,6 +228,31 @@ class BasicArgumentParser(argparse.ArgumentParser):
help="whether to disable optimization of the RTN mode(iters=0) (default is False).",
)
+ ## ======================= MLLM =======================
+ self.add_argument(
+ "--quant_nontext_module",
+ action="store_true",
+ help="whether to quantize non-text module, e.g. vision component",
+ )
+
+ self.add_argument(
+ "--extra_data_dir",
+ default=None,
+ type=str,
+ help="dataset dir for storing images/audio/videos. "
+ "Can be a dir path or multiple dir path with format as "
+ "'image=path_to_image,video=path_to_video,audio=path_to_audio'"
+ "By default, it will search in the relative path, "
+ "and if not find, will automatic download.",
+ )
+
+ self.add_argument(
+ "--template",
+ default=None,
+ type=str,
+ help="the template for building training dataset. It can be a custom one.",
+ )
+
class EvalArgumentParser(argparse.ArgumentParser):
@@ -234,7 +261,7 @@ class EvalArgumentParser(argparse.ArgumentParser):
self.add_argument(
"--model", "--model_name", "--model_name_or_path", default="facebook/opt-125m", help="model name or path"
)
- self.add_argument("--mllm", default=False, help="whether to eval multi-modal model.")
+ self.add_argument("--mllm", action="store_true", help="whether to eval multi-modal model.")
self.add_argument(
"--device",
"--devices",
@@ -419,23 +446,46 @@ def tune(args):
model_name = model_name[:-1]
logger.info(f"start to quantize {model_name}")
- from auto_round.utils import llm_load_model
+ from auto_round.utils import llm_load_model, mllm_load_model
- model, tokenizer, low_cpu_mem_usage = llm_load_model(
- model_name,
- trust_remote_code=not args.disable_trust_remote_code,
- device=args.device,
- low_cpu_mem_mode=args.low_cpu_mem_mode,
- low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,
- model_dtype=args.model_dtype,
- )
+ if args.mllm:
+ model, processor, tokenizer, image_processor = mllm_load_model(
+ model_name,
+ device=device_str,
+ use_auto_mapping=use_auto_mapping,
+ trust_remote_code=not args.disable_trust_remote_code,
+ model_dtype=args.model_dtype,
+ )
+ low_cpu_mem_usage = False
+ else:
+ model, tokenizer, low_cpu_mem_usage = llm_load_model(
+ model_name,
+ use_auto_mapping=use_auto_mapping,
+ trust_remote_code=not args.disable_trust_remote_code,
+ device=device_str,
+ low_cpu_mem_mode=args.low_cpu_mem_mode,
+ low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir,
+ model_dtype=args.model_dtype,
+ )
- from auto_round import AutoRound, AutoRoundAdam
+ from auto_round import AutoRound, AutoRoundAdam, AutoRoundMLLM
if "bloom" in model_name:
args.low_gpu_mem_usage = False
- round = AutoRound if not args.adam else AutoRoundAdam
+ round = AutoRound
+ mllm_kwargs = {}
+ if args.adam:
+ round = AutoRoundAdam
+ if args.mllm:
+ round = AutoRoundMLLM
+ mllm_kwargs = {
+ "processor": processor,
+ "image_processor": image_processor,
+ "extra_data_dir": args.extra_data_dir,
+ "quant_nontext_module": args.quant_nontext_module,
+ "template": args.template,
+ }
layer_config = {}
not_quantize_layer_names = get_fp_layer_names(model, args.fp_layers)
@@ -485,10 +535,10 @@ def tune(args):
enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False
autoround = round(
- model,
- tokenizer,
- args.bits,
- args.group_size,
+ model=model,
+ tokenizer=tokenizer,
+ bits=args.bits,
+ group_size=args.group_size,
sym=not args.asym,
batch_size=args.batch_size,
dataset=args.dataset,
@@ -521,6 +571,7 @@ def tune(args):
super_group_size=args.super_group_size,
super_bits=args.super_bits,
disable_opt_rtn=args.disable_opt_rtn,
+ **mllm_kwargs,
)
model_name = args.model.rstrip("/")
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index c381854a9..9c3933610 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -28,6 +28,8 @@ SUPPORT_ONLY_TEXT_MODELS = [
"internvl_chat",
]
+NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]
+
SPECIAL_SHARED_CACHE_KEYS = {
"Gemma3ForConditionalGeneration": ("position_embeddings_global", "position_embeddings_local")
}
diff --git a/auto_round/utils.py b/auto_round/utils.py
index dc738c122..0c00754e4 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1379,6 +1379,7 @@ def llm_load_model(
def mllm_load_model(
pretrained_model_name_or_path,
+ device="cpu",
torch_dtype="auto",
use_auto_mapping=True,
trust_remote_code=True,
@@ -1391,6 +1392,10 @@ def mllm_load_model(
from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
from transformers import AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
+ device_str, use_auto_mapping = get_device_and_parallelism(device)
+ torch_dtype = "auto"
+ if device_str is not None and "hpu" in device_str:
+ torch_dtype = torch.bfloat16
if os.path.isdir(pretrained_model_name_or_path):
config = json.load(open(os.path.join(pretrained_model_name_or_path, "config.json")))
else:
@@ -1455,12 +1460,21 @@ def mllm_load_model(
torch_dtype=torch_dtype,
device_map="auto" if use_auto_mapping else None,
)
- tokenizer = AutoTokenizer.from_pretrained(
- pretrained_model_name_or_path, trust_remote_code=trust_remote_code
- )
- processor = AutoProcessor.from_pretrained(
- pretrained_model_name_or_path, trust_remote_code=trust_remote_code
- )
+
+ if "Mistral-Small-3.2" in pretrained_model_name_or_path:
+ from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # pylint: disable=E0401
+
+ if os.path.isdir(pretrained_model_name_or_path):
+ tokenizer = MistralTokenizer.from_file(os.path.join(pretrained_model_name_or_path, "tekken.json"))
+ else:
+ tokenizer = MistralTokenizer.from_hf_hub(pretrained_model_name_or_path)
+ else:
+ tokenizer = AutoTokenizer.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code
+ )
+ processor = AutoProcessor.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code
+ )
try:
from transformers import AutoImageProcessor
diff --git a/test/test_cpu/test_basic_usage.py b/test/test_cpu/test_basic_usage.py
index 7de1204c2..93266c2b7 100644
--- a/test/test_cpu/test_basic_usage.py
+++ b/test/test_cpu/test_basic_usage.py
@@ -44,29 +44,25 @@ class TestAutoRoundCmd(unittest.TestCase):
assert False, "cmd line test fail, please have a check"
# test mllm script
- # test auto_round_mllm help
- res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm -h")
- if res > 0 or res == -1:
- assert False, "cmd line test fail, please have a check"
# test auto_round_mllm --eval help
- res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm --eval -h")
+ res = os.system(f"cd ../.. && {python_path} -m auto_round --eval -h")
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
# test auto_round_mllm --lmms help
- res = os.system(f"cd ../.. && {python_path} -m auto_round --mllm --lmms -h")
+ res = os.system(f"cd ../.. && {python_path} -m auto_round --eval --lmms -h")
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
res = os.system(
- f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved"
+ f"cd ../.. && {python_path} -m auto_round --mllm --model Qwen/Qwen2-VL-2B-Instruct --iter 2 --nsamples 10 --seqlen 32 --format auto_round --output_dir ./saved"
)
if res > 0 or res == -1:
assert False, "cmd line test fail, please have a check"
res = os.system(
- f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --seqlen 256 --format auto_round"
+ f"cd ../.. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --model Qwen/Qwen2-VL-2B-Instruct --seqlen 256 --format auto_round"
" --quant_nontext_module --output_dir ./saved "
)
if res > 0 or res == -1: