PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
patch 1 for mllm (#298)
n1ck-guo73861bb76bd8
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 2401a732b..621e8b877 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -55,7 +55,6 @@ from .utils import ()from .low_cpu_mem.utils import get_layers_before_block-class AutoRound(object):"""For more information, please refer to Cheng, Wenhua, et al. "Optimize weight rounding via signed gradient descentfor the quantization of llms." arXiv preprint arXiv:2309.05516 (2023).diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 2aebd7565..fa20a04c0 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -13,6 +13,7 @@# limitations under the License.from typing import Optional, Union+from tqdm import tqdmimport torch@@ -27,6 +28,8 @@ from .template import get_template, Templatefrom .mllm_dataset import get_mllm_dataloaderfrom ..low_cpu_mem.utils import get_layers_before_blockfrom ..special_model_handler import check_mllm_model_batch++class AutoRoundMLLM(AutoRound):"""Class for automatic rounding-based quantization with MLLMs.@@ -74,6 +77,7 @@ class AutoRoundMLLM(AutoRound):self,model,tokenizer,+ image_processor = None,bits: int = 4,group_size: int = 128,sym: bool = False,@@ -83,7 +87,7 @@ class AutoRoundMLLM(AutoRound):device: str = None,lr_scheduler=None,dataset: Union[str, list, tuple, torch.utils.data.DataLoader] = None,- extra_data_dir: Union[str, torch.utils.data.DataLoader] = None,+ extra_data_dir: str = None,template: Union[str, Template] = None,quant_nontext_module: bool = False,enable_quanted_input: bool = True,@@ -115,13 +119,12 @@ class AutoRoundMLLM(AutoRound):quant_block_list = get_multimodal_block_names(model, quant_nontext_module)self.extra_data_dir = extra_data_dirself.quant_nontext_module = quant_nontext_module- self.template = template- if self.template is None:- self.template = get_template(model.config.model_type)+ self.image_processor = image_processor+ self.template = template if template is not None else model.config.model_type+ self.template = get_template(+ self.template, model=model, tokenizer=tokenizer, image_processor=image_processor)assert dataset is not None, "dataset should not be None"batch_size, gradient_accumulate_steps = check_mllm_model_batch(model, batch_size, gradient_accumulate_steps)- if isinstance(dataset, str):- dataset = get_mllm_dataloader(self.template, model, tokenizer, dataset, extra_data_dir, seqlen, batch_size)super(AutoRoundMLLM, self).__init__(model=model,@@ -177,7 +180,16 @@ class AutoRoundMLLM(AutoRound):if isinstance(self.dataset, str):dataset = self.dataset.replace(" ", "")self.dataloader = get_mllm_dataloader(- self.template, self.model, self.tokenizer, dataset, self.extra_data_dir, self.seqlen, bs)+ template=self.template,+ model=self.model,+ tokenizer=self.tokenizer,+ image_processor=self.image_processor,+ dataset=dataset,+ extra_data_dir=self.extra_data_dir,+ seqlen=self.seqlen,+ bs=bs,+ seed=self.seed+ )else:self.dataloader = self.datasettotal_cnt = 0@@ -187,7 +199,7 @@ class AutoRoundMLLM(AutoRound):for n, m in embed_layers:m = m.to(self.device)- for data in self.dataloader:+ for data in tqdm(self.dataloader, desc="calib", total=nsamples-1):if data is None:continueif isinstance(data, torch.Tensor):@@ -199,8 +211,6 @@ class AutoRoundMLLM(AutoRound):exit()# data = self.template._encode(data)data = self.template.processor.get_input(- self.model,- self.tokenizer,text=data,images=None,max_length=self.seqlen,@@ -215,14 +225,9 @@ class AutoRoundMLLM(AutoRound):if isinstance(text, dict):text = [text]input_text = self.template._encode(text)- image = None- if "image" in data:- image = self.template.processor.image_processor(data["image"])data = self.template.processor.get_input(- self.model,- self.tokenizer,text=input_text,- images=image,+ images=data["image"],max_length=self.seqlen,squeeze=False,)@@ -243,9 +248,6 @@ class AutoRoundMLLM(AutoRound):data_new[key] = to_dtype(data_new[key], self.model.dtype)input_ids = data_new["input_ids"]- if input_ids.shape[-1] < self.seqlen:- continue-try:if isinstance(data_new, torch.Tensor):self.model(data_new)diff --git a/auto_round/mllm/eval.py b/auto_round/mllm/eval.pyindex ec1b2b118..58f8ba182 100644--- a/auto_round/mllm/eval.py+++ b/auto_round/mllm/eval.py@@ -27,6 +27,7 @@# limitations under the License.import os+import timeimport jsonfrom functools import partial@@ -84,6 +85,10 @@ def mllm_eval(model = Noneif data_store_dir is not None:+ if not os.path.exists(data_store_dir):+ oldmask = os.umask(000)+ os.makedirs(data_store_dir, mode=0o777)+ os.umask(oldmask)os.environ['LMUData'] = data_store_dirmodel_name = pretrained_model_name_or_path@@ -119,7 +124,10 @@ def mllm_eval(pred_root = os.path.join(work_dir, model_name)os.makedirs(pred_root, exist_ok=True)+ st = time.time()+ rt_file = open(f'{pred_root}/{model_name}_eval_cost.txt', 'w')for dataset_name in dataset:+ task_st = time.time()try:dataset_kwargs = {}if dataset_name in ['MMLongBench_DOC', 'DUDE', 'DUDE_MINI', 'SLIDEVQA', 'SLIDEVQA_MINI']:@@ -269,8 +277,10 @@ def mllm_eval(logger.info('\n' + tabulate.tabulate(eval_results))except:logger.info(eval_results.to_string())-+ rt_file.write('%s cost: %.4fs\n' % (dataset_name, time.time() - task_st))except Exception as e:logger.exception(f'Model {model_name} x Dataset {dataset_name} combination failed: {e}, ''skipping this combination.')continue+ rt_file.write('%d tasks cost: %.4fs\n' % (len(dataset), time.time() - st))+ rt_file.close()diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.pyindex 423d70e0d..cf319fbf0 100644--- a/auto_round/mllm/mllm_dataset.py+++ b/auto_round/mllm/mllm_dataset.py@@ -19,9 +19,11 @@ from typing import Dictimport torchfrom torch.utils.data import Dataset, DataLoader+from transformers import set_seedfrom .utils import _extract_data_dirfrom .template import Template+from ..utils import loggerMLLM_DATASET : Dict[str, Dataset] = {}@@ -44,9 +46,17 @@ def register_dataset(name):return register-@register_dataset("llava")++@register_dataset("liuhaotian/llava")class LlavaDataset(Dataset):"""Dataset for supervised fine-tuning."""+ BASE_LLAVA_URL = "https://huggingface.co/datasets/liuhaotian/LLaVA-Instruct-150K/resolve/main/"+ LLAVA_DATASET = {+ "llava_conv_58k": BASE_LLAVA_URL + "conversation_58k.json?download=true",+ "llava_instruct_80k": BASE_LLAVA_URL + "llava_instruct_80k.json?download=true",+ "llava_instruct_150k": BASE_LLAVA_URL + "llava_instruct_150k.json?download=true",+ }+ _COCO_DATA_URL = "http://images.cocodataset.org/train2017/"def __init__(self,@@ -54,8 +64,8 @@ class LlavaDataset(Dataset):model,tokenzier,dataset_path,- extra_data_dir,- max_length,+ extra_data_dir=None,+ seqlen=None,padding=True,truncation=True,) -> None:@@ -64,13 +74,44 @@ class LlavaDataset(Dataset):self.model_type = template.model_typeself.template = templateself.tokenizer = tokenzier- self.questions = json.load(open(dataset_path, "r"))+ if os.path.exists(dataset_path):+ logger.info(f'use dataset {dataset_path}, loading from disk...')+ self.questions = json.load(open(dataset_path, "r"))+ else:+ import requests+ dataset_name = dataset_path.split('/')[-1]+ if dataset_name in self.LLAVA_DATASET:+ logger.info(f'use dataset {dataset_name}, downloading ...')+ self.questions = requests.get(self.LLAVA_DATASET[dataset_name], stream=True).json()+ else:+ raise KeyError(f"{dataset_path} is not support, we support {self.LLAVA_DATASET.keys()}.")+ self.seqlen = seqlen+ self.questions = self.check(self.questions, seqlen)self.padding = paddingself.truncation = truncationself.extra_data_dir = extra_data_dir- self.max_length = max_lengthself.role_mapping = {"human": "user", "gpt": "assistant"}self.cached_data_dict = {}++ self.image_fold = None+ if extra_data_dir is not None:+ image_fold = _extract_data_dir(self.extra_data_dir)+ if isinstance(image_fold, dict):+ image_fold = image_fold['image']+ self.image_fold = image_fold+++ @staticmethod+ def check(questions, seqlen):+ new_questions = []+ for source in questions:+ text_lenght = 0+ for text in source['conversations']:+ text_lenght += len(text['value'].split(' '))+ if text_lenght >= seqlen:+ new_questions.append(source)+ assert len(new_questions) > 0, f"no data with length greater than {seqlen}, please check"+ return new_questionsdef __len__(self):@@ -81,28 +122,30 @@ class LlavaDataset(Dataset):return self.cached_data_dict[i]text = self.questions[i]["conversations"]- text = self.covert_conversations(text)+ if self.template.model_type != "llava":+ text = self.covert_conversations(text)- text = self.template._encode(text)- if self.extra_data_dir is not None:- image_fold = _extract_data_dir(self.extra_data_dir)- if isinstance(image_fold, dict):- image_fold = image_fold['image']+ if self.image_fold is not None:image_path = os.path.join(- image_fold, os.path.basename(self.questions[i]["image"]))+ self.image_fold, os.path.basename(self.questions[i]["image"]))else:image_path = self.questions[i]["image"]- image = self.template.processor.image_processor(image_path)+ if not os.path.exists(image_path):+ image_path = self._COCO_DATA_URL + self.questions[i]["image"].split('/')[-1]+ # image = self.template.processor.image_processor(image_path)+ text = self.template._encode(text)++ max_length = self.seqlen+ truncation_strategy = "text"ret = self.template.processor.get_input(- self.model,- self.tokenizer,text=text,- images=image,+ images=image_path,padding=self.padding,truncation=self.truncation,return_tensors="pt",- max_length = self.max_length+ max_length = max_length,+ truncation_strategy=truncation_strategy)self.cached_data_dict[i] = retreturn ret@@ -111,8 +154,9 @@ class LlavaDataset(Dataset):new_data = []for d in data:content = d["value"]- for old, new in self.template.replace_tokens:- content = content.replace(old, new)+ if self.template.replace_tokens is not None:+ for old, new in self.template.replace_tokens:+ content = content.replace(old, new)new_data.append({"role": self.role_mapping.get(d["from"], d["from"]),"content": content@@ -124,12 +168,14 @@ def get_mllm_dataloader(template,model,tokenizer,- dataset,- extra_data_dir,+ image_processor=None,+ dataset="liuhaotian/llava_conv_58k",+ extra_data_dir=None,seqlen=512,bs=1,split=None,apply_template=None,+ seed=42,):"""Generate a DataLoader for calibration using specified parameters.@@ -148,13 +194,19 @@ def get_mllm_dataloader(Returns:DataLoader: The DataLoader for the calibrated datasets."""- assert isinstance(template, Template)+ if isinstance(template, str):+ from .template import get_template+ template = get_template(template, model=model, tokenizer=tokenizer, image_processor=image_processor)if isinstance(dataset, str):if os.path.isfile(dataset):- dataset = MLLM_DATASET['llava'](+ dataset = MLLM_DATASET['liuhaotian/llava'](+ template, model, tokenizer, dataset, extra_data_dir,+ seqlen=min(seqlen, tokenizer.model_max_length))+ elif "liuhaotian/llava" in dataset:+ dataset = MLLM_DATASET["liuhaotian/llava"](template, model, tokenizer, dataset, extra_data_dir,- max_length=min(seqlen, tokenizer.model_max_length))+ seqlen=min(seqlen, tokenizer.model_max_length))else:from datasets import load_datasetfrom ..calib_dataset import get_tokenizer_function@@ -163,8 +215,10 @@ def get_mllm_dataloader(dataset = dataset.map(tokenizer_function, batched=True)+ set_seed(seed)dataloader_params = {"batch_size": bs,+ "shuffle": True,"collate_fn": dataset.template.processor.data_collator}diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.pyindex eb6305fcf..3b285e015 100644--- a/auto_round/mllm/processor.py+++ b/auto_round/mllm/processor.py@@ -29,39 +29,66 @@ def regist_processor(name):class BasicProcessor:def __init__(self):pass++ def post_init(self, model, tokenizer, image_processor=None, **kwargs):+ self.model = model+ self.tokenizer = tokenizer+ if image_processor is not None:+ self.image_processor = image_processor+ else:+ self.image_processor = self.default_image_processordef get_input(self,- model,- tokenizer,text,images,- padding=True,- truncation=True,return_tensors="pt",- max_length=None,squeeze=True,+ max_length=None,+ truncation_strategy="text",**kwargs):++ if isinstance(text, list):+ for message in text:+ if not ("role" in message and "content" in message):+ raise ValueError(+ "When passing chat dicts as input, each dict must have a 'role' and 'content' key.")+ continue_final_message = text[-1]["role"] == "assistant"- if max_length:- token_length = len(tokenizer(text).input_ids)- if token_length < max_length:- if tokenizer.pad_token:- text += tokenizer.pad_token * (max_length - token_length)- else:- text = tokenizer.decode(tokenizer(text).input_ids[:max_length])+ # add_generation_prompt=True will add <|im_start|>assistant to the end+ try:+ text = self.tokenizer.apply_chat_template(+ text, tokenize=False, add_generation_prompt=not continue_final_message,+ continue_final_message=continue_final_message,)+ except:+ raise NotImplementedError("current not support for non-instructed model.")++ if text == '':+ raise NotImplementedError("current not support for non-instructed model.")++ if images is not None:+ images = self.image_processor(images)++ if truncation_strategy == "text" and max_length is not None:+ text = text[:max_length]- ret = tokenizer.processor(+ ret = self.tokenizer.processor(text=text,images=images,- padding=padding,- truncation=truncation,return_tensors=return_tensors,# videos = None)- if squeeze:+ if truncation_strategy == "token" and max_length:+ seqlen = ret['input_ids'].shape[-1]for key in ret:- ret[key] = ret[key][0]+ shape_ = ret[key].shape+ if len(shape_) == 2 and shape_[-1] == seqlen:+ ret[key] = ret[key][:,:max_length]+ elif len(shape_) == 4 and shape_[1] == seqlen:+ ret[key] = ret[key][:,:max_length]++ if squeeze:+ ret = self.squeeze_result(ret)return ret@staticmethod@@ -69,57 +96,41 @@ class BasicProcessor:return default_data_collator(batch)@staticmethod- def image_processor(image_path_or_url):+ def default_image_processor(image_path_or_url):return fetch_image(image_path_or_url)++ @staticmethod+ def squeeze_result(ret):+ for key in ret:+ ret[key] = ret[key][0]+ return ret@regist_processor("qwen2_vl")class Qwen2VLProcessor(BasicProcessor):- def get_input(- self,- model,- tokenizer,- text,- images,- padding=True,- truncation=True,- return_tensors="pt",- max_length=None,- squeeze=True,- **kwargs):-- if max_length:- token_length = len(tokenizer(text).input_ids)- if token_length < max_length:- if tokenizer.pad_token:- text += tokenizer.pad_token * (max_length - token_length)- else:- text = tokenizer.decode(tokenizer(text).input_ids[:max_length])-- ret = tokenizer.processor(- text=text,- images=images,- padding=padding,- truncation=truncation,- return_tensors=return_tensors,- # videos = None- )- if squeeze:- for key in ret:- if key == "pixel_values":+ @staticmethod+ def squeeze_result(ret):+ for key in ret:+ if key == "pixel_values":continue- ret[key] = ret[key][0]+ ret[key] = ret[key][0]return ret-@regist_processor("cogvlm2")class CogVLM2Processor(BasicProcessor):def get_input(- self, model, tokenizer, text, images, max_length=2048,- padding=True, truncation=True, squeeze=True, **kwargs):+ self, text, images,+ squeeze=True, **kwargs):++ if images is not None:+ images = self.image_processor(images)+padding_len = 2303+ max_length = 0max_length += padding_len- input_data = model.build_conversation_input_ids(- tokenizer,+ truncation = True+ padding = False+ input_data = self.model.build_conversation_input_ids(+ self.tokenizer,query=text,history=None,images=[images],@@ -177,5 +188,51 @@ class CogVLM2Processor(BasicProcessor):return batched_data@staticmethod- def image_processor(image_path_or_url):- return fetch_image(image_path_or_url).convert('RGB')\ No newline at end of file+ def default_image_processor(image_path_or_url):+ return fetch_image(image_path_or_url).convert('RGB')+++from ..utils import LazyImport+llava_train = LazyImport("llava.train.train")++@regist_processor("llava")+class LlavaProcessor(BasicProcessor):+ def post_init(self, model, tokenizer, image_processor=None, **kwargs):+ self.model = model+ self.tokenizer = tokenizer+ assert image_processor is not None, "for llava model, image_processor should not be None"+ self.image_processor = image_processor+ self.collator_func = llava_train.DataCollatorForSupervisedDataset(tokenizer=self.tokenizer)+++ def get_input(+ self, text, images,max_length=None,+ squeeze=True, truncation_strategy="text", **kwargs):++ if images is not None:+ images = fetch_image(images).convert('RGB')+ images = self.image_processor.preprocess(images, return_tensors='pt')['pixel_values'][0]++ class DataArgs:+ is_multimodal = True+ mm_use_im_start_end = False++ if truncation_strategy == "text" and max_length is not None:+ text = text[:max_length]++ input_data = llava_train.preprocess_multimodal([text], DataArgs())+ ret = llava_train.preprocess(input_data, self.tokenizer, has_image=(images is not None))++ if truncation_strategy == "token" and max_length:+ seqlen = ret['input_ids'].shape[-1]+ for key in ret:+ if ret[key].shape[-1] == seqlen:+ ret[key] = ret[key][:,:max_length]+ if squeeze:+ ret = self.squeeze_result(ret)+ ret['image'] = images++ return ret++ def data_collator(self, batch):+ return self.collator_func(batch)\ No newline at end of filediff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex 3478f1a10..6d477f04f 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -19,8 +19,6 @@ from typing import Dict, Optional, List, Union, Sequencefrom enum import Enum, uniquefrom ..utils import logger--from .processor import BasicProcessor, PROCESSORSTEMPLATES: Dict[str, "Template"] = {}@@ -50,26 +48,30 @@ class Template:format_separator: strdefault_system: strreplace_tokens: List[tuple]+ add_special_token: boolprocessor: "BasicProcessor"def _encode(self, sources):"""Encodes formatted inputs to pairs of token ids."""- element = ""- for i, source in enumerate(sources):- if i == 0:- element += fill_content(self.format_system, content=self.default_system)- # if i > 0 and i % 2 ==0:- # element += fill_content(self.format_separator)-- if source['role'] == Role.USER.value:- element += fill_content(self.format_user, content=source["content"])- elif source['role'] == Role.ASSISTANT.value:- element += fill_content(self.format_assistant, content=source["content"])- elif source['role'] == Role.OBSERVATION.value:- element += fill_content(self.format_observation, content=source["content"])- elif source['role'] == Role.FUNCTION.value:- element += fill_content(self.format_function, content=source["content"])- return element+ if self.add_special_token:+ element = ""+ for i, source in enumerate(sources):+ if i == 0:+ element += fill_content(self.format_system, content=self.default_system)+ # if i > 0 and i % 2 ==0:+ # element += fill_content(self.format_separator)++ if source['role'] == Role.USER.value:+ element += fill_content(self.format_user, content=source["content"])+ elif source['role'] == Role.ASSISTANT.value:+ element += fill_content(self.format_assistant, content=source["content"])+ elif source['role'] == Role.OBSERVATION.value:+ element += fill_content(self.format_observation, content=source["content"])+ elif source['role'] == Role.FUNCTION.value:+ element += fill_content(self.format_function, content=source["content"])+ return element+ else:+ return sourcesdef _register_template(@@ -82,7 +84,8 @@ def _register_template(format_separator: Optional[str] = None,default_system: str = "",replace_tokens: List[tuple] = None,- processor: "BasicProcessor" = PROCESSORS["basic"]+ add_special_token: Optional[bool] = True,+ processor: "BasicProcessor" = PROCESSORS["basic"],):"""Registers a chat template."""template_class = Template@@ -102,6 +105,7 @@ def _register_template(format_separator = format_separator or default_format_separator,default_system = default_system,replace_tokens = replace_tokens,+ add_special_token = add_special_token,processor = processor())return TEMPLATES[model_type]@@ -137,7 +141,7 @@ def _load_preset_template():_load_preset_template()-def get_template(template_or_path: str):+def get_template(template_or_path: str, model=None, tokenizer=None, image_processor=None):"""Get template by template name or from a json file.Args:@@ -155,4 +159,7 @@ def get_template(template_or_path: str):else:logger.warning(f"Unable to recognize {template_or_path}, using default template instead.")template = TEMPLATES["default"]++ template.processor.post_init(model=model, tokenizer=tokenizer, image_processor=image_processor)+return template\ No newline at end of filediff --git a/auto_round/mllm/templates/default.json b/auto_round/mllm/templates/default.jsonindex c16844529..51ec37f34 100644--- a/auto_round/mllm/templates/default.json+++ b/auto_round/mllm/templates/default.json@@ -8,5 +8,6 @@"format_separator": "\n","default_system": "You are a helpful assistant.","replace_tokens": null,- "processor": "basic"+ "processor": "basic",+ "add_special_token" : false}\ No newline at end of filediff --git a/auto_round/mllm/templates/llava.json b/auto_round/mllm/templates/llava.jsonnew file mode 100644index 000000000..13d5a6b49--- /dev/null+++ b/auto_round/mllm/templates/llava.json@@ -0,0 +1,6 @@+{+ "model_type": "llava",+ "replace_tokens": null,+ "processor": "llava",+ "add_special_token" : false+}\ No newline at end of filediff --git a/auto_round/mllm/templates/phi3_v.json b/auto_round/mllm/templates/phi3_v.jsonnew file mode 100644index 000000000..09949ff47--- /dev/null+++ b/auto_round/mllm/templates/phi3_v.json@@ -0,0 +1,6 @@+{+ "model_type": "phi3_v",+ "replace_tokens": ["<image>", "<|image_1|>"],+ "processor": "basic",+ "add_special_token" : false+}\ No newline at end of filediff --git a/auto_round/mllm/utils.py b/auto_round/mllm/utils.pyindex ce3bcae1f..118ac1974 100644--- a/auto_round/mllm/utils.py+++ b/auto_round/mllm/utils.py@@ -24,7 +24,7 @@ from PIL import Image # pylint: disable=E0401def _extract_data_dir(dir_path: str):if os.path.isdir(dir_path):return dir_path- else:+ elif "=" in dir_path:result = {}dir_path = dir_path.split(",")for _path in dir_path:@@ -32,6 +32,8 @@ def _extract_data_dir(dir_path: str):if k in ['image', 'video', 'audio']:result[k] = vreturn result+ else:+ raise TypeError("incorrect input of extra_data_dir, please use auto_round --help for more details.")def fetch_image(path_or_url):diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 4690ccab9..54a25f3a2 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -30,7 +30,7 @@ class BasicArgumentParser(argparse.ArgumentParser):def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)self.add_argument("--model", "--model_name", "--model_name_or_path",- default="facebook/opt-125m",+ default="Qwen/Qwen2-VL-2B-Instruct",help="model name or path")self.add_argument('--eval', action='store_true',@@ -50,8 +50,10 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--asym", action='store_true',help="whether to use asym quantization")- self.add_argument("--dataset", type=str, default=None,- help="the dataset for quantization training. It can be a custom one.")+ self.add_argument("--dataset", type=str, default="liuhaotian/llava_conv_58k",+ help="the dataset for quantization training."+ " current support llava_conv_58k,llava_instruct_80k "+ "It can be a custom one.")self.add_argument("--lr", default=None, type=float,help="learning rate, if None, it will be set to 1.0/iters automatically")@@ -65,7 +67,7 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--adam", action='store_true',help="whether to use adam optimizer instead of SignSGD")- self.add_argument("--gradient_accumulate_steps", default=1, type=int,+ self.add_argument("--gradient_accumulate_steps", default=4, type=int,help="gradient accumulate steps")self.add_argument("--nblocks", default=1, type=int,@@ -136,17 +138,19 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--quant_nontext_module", action='store_true',help="whether to quantize non-text module, e.g. vision component")- self.add_argument("--extra_data_dir", default="", type=str,+ self.add_argument("--extra_data_dir", default=None, type=str,help="dataset dir for storing images/audio/videos. ""Can be a dir path or multiple dir path with format as ""'image=path_to_image,video=path_to_video,audio=path_to_audio'"- "By default, it will search in the relative path.")+ "By default, it will search in the relative path, "+ "and if not find, will automatic download.")self.add_argument("--template", default=None, type=str,help="the template for building training dataset. It can be a custom one.")## ======================= VLM eval=======================- self.add_argument("--tasks", type=str, default="COCO_VAL",+ self.add_argument("--tasks", type=str,+ default="MMBench_DEV_EN_V11,ScienceQA_VAL,TextVQA_VAL,POPE",help="eval tasks for VLMEvalKit.")# Args that only apply to Video Datasetself.add_argument("--nframe", type=int, default=8,@@ -189,7 +193,7 @@ def setup_parser():parser.add_argument("--iters", "--iter", default=200, type=int,help=" iters")- parser.add_argument("--seqlen", "--seq_len", default=2048, type=int,+ parser.add_argument("--seqlen", "--seq_len", default=256, type=int,help="sequence length")parser.add_argument("--nsamples", default=128, type=int,@@ -250,24 +254,30 @@ def tune(args):torch_dtype = torch.bfloat16# load_model- config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)- tokenizer = AutoTokenizer.from_pretrained(model_name)- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)- tokenizer.processor = processor- model_type = config.model_type- if "qwen2_vl" in model_type:- from transformers import Qwen2VLForConditionalGeneration- cls = Qwen2VLForConditionalGeneration- elif "mllama" in model_type:- from transformers import MllamaForConditionalGeneration- cls = MllamaForConditionalGeneration+ processor, image_processor = None, None+ if "llava" in model_name:+ from llava.model.builder import load_pretrained_model # pylint: disable=E0401+ tokenizer, model, image_processor, _ = load_pretrained_model(model_name, model_base=None, model_name=model_name,+ torch_dtype=torch_dtype)+ model_type = "llava"else:- cls = AutoModelForCausalLM-+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)+ tokenizer = AutoTokenizer.from_pretrained(model_name)+ processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)+ tokenizer.processor = processor+ model_type = config.model_type+ if "qwen2_vl" in model_type:+ from transformers import Qwen2VLForConditionalGeneration+ cls = Qwen2VLForConditionalGeneration+ elif "mllama" in model_type:+ from transformers import MllamaForConditionalGeneration+ cls = MllamaForConditionalGeneration+ else:+ cls = AutoModelForCausalLM+model = cls.from_pretrained(- model_name, trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype,+ model_name,trust_remote_code=not args.disable_trust_remote_code, torch_dtype=torch_dtype,device_map="auto" if use_auto_mapping else None)-if "cogvlm2" in model_name:model.config.model_type = "cogvlm2"@@ -276,6 +286,12 @@ def tune(args):model = model.eval()seqlen = args.seqlen+ if args.model_dtype != None:+ if args.model_dtype == "float16" or args.model_dtype == "fp16":+ model = model.to(torch.float16)+ if args.model_dtype == "bfloat16" or args.model_dtype == "bfp16":+ model = model.to(torch.bfloat16)+round = AutoRoundMLLMlayer_config = {}for n, m in model.named_modules():@@ -310,14 +326,15 @@ def tune(args):if args.quant_lm_head and args.low_gpu_mem_usage:print(f"warning, low_gpu_mem_usage=False is strongly recommended if the whole model could be loaded to "f"gpu")-- autoround = round(model, tokenizer, dataset=args.dataset, extra_data_dir=args.extra_data_dir,- bits=args.bits, group_size=args.group_size, sym=not args.asym,- batch_size=args.batch_size, seqlen=seqlen, nblocks=args.nblocks, iters=args.iters,- lr=args.lr, minmax_lr=args.minmax_lr, enable_quanted_input=not args.disable_quanted_input,- amp=not args.disable_amp, nsamples=args.nsamples, low_gpu_mem_usage=args.low_gpu_mem_usage,++ autoround = round(model, tokenizer, image_processor=image_processor, dataset=args.dataset,+ extra_data_dir=args.extra_data_dir, bits=args.bits, group_size=args.group_size,+ sym=not args.asym, batch_size=args.batch_size, seqlen=seqlen, nblocks=args.nblocks,+ iters=args.iters, lr=args.lr, minmax_lr=args.minmax_lr, amp=not args.disable_amp,+ enable_quanted_input=not args.disable_quanted_input,+ nsamples=args.nsamples, low_gpu_mem_usage=args.low_gpu_mem_usage,device=device_str, seed=args.seed, gradient_accumulate_steps=args.gradient_accumulate_steps,- scale_dtype=args.scale_dtype, layer_config=layer_config,+ scale_dtype=args.scale_dtype, layer_config=layer_config, template=args.template,enable_minmax_tuning=not args.disable_minmax_tuning, act_bits=args.act_bits,quant_nontext_module=args.quant_nontext_module, not_use_best_mse=args.not_use_best_mse)model, _ = autoround.quantize()@@ -335,12 +352,18 @@ def tune(args):inplace = False if len(format_list) > 1 else Truefor format_ in format_list:eval_folder = f'{export_dir}-{format_}'- autoround.save_quantized(eval_folder, format=format_, inplace=inplace, processor=processor)+ if processor is not None and not hasattr(processor, "chat_template"):+ processor.chat_template = None+ safe_serialization = True+ if "phi3_v" in model_type:+ safe_serialization = False+ autoround.save_quantized(+ eval_folder, format=format_, inplace=inplace, processor=processor, safe_serialization=safe_serialization)def eval(args):if isinstance(args.tasks, str):- args.tasks = args.tasks.split(',')+ args.tasks = args.tasks.replace(' ', '').split(',')from auto_round.mllm import mllm_evalmllm_eval(args.model,diff --git a/test/test_basic_usage.py b/test/test_basic_usage.pyindex 0ee08c9b6..9664b2eb3 100644--- a/test/test_basic_usage.py+++ b/test/test_basic_usage.py@@ -3,6 +3,8 @@ import shutilimport sysimport unittest+sys.path.insert(0, '..')+class TestAutoRoundCmd(unittest.TestCase):@classmethod@@ -13,11 +15,39 @@ class TestAutoRoundCmd(unittest.TestCase):def tearDownClass(self):shutil.rmtree("./saved", ignore_errors=True)shutil.rmtree("runs", ignore_errors=True)-+def test_auto_round_cmd(self):python_path = sys.executable+ # test llm script+ res = os.system(+ f"cd .. && {python_path} -m auto_round -h")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"++ res = os.system(+ f"cd .. && {python_path} -m auto_round --model 'facebook/opt-125m' --iter 2 --nsamples 1 --format auto_gptq,auto_round --disable_eval --output_dir ./saved")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+++ # test mllm script+ res = os.system(+ f"cd .. && {python_path} -m auto_round --mllm -h")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+res = os.system(- f"{python_path} ../auto_round/__main__.py --model 'facebook/opt-125m' --iter 2 --nsamples 1 --format auto_gptq,auto_round --disable_eval --output_dir ./saved")+ f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --format auto_round --output_dir ./saved")if res > 0 or res == -1:- assert False, "cmd line test fail, please have acheck"+ assert False, "cmd line test fail, please have a check"++ res = os.system(+ f"cd .. && {python_path} -m auto_round --mllm --iter 2 --nsamples 10 --format auto_round"+ " --quant_nontext_module --output_dir ./saved")+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+++if __name__ == "__main__":+ unittest.main()\ No newline at end of file