PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
save processor automatically (#372)
Heng Guoe4528e92b49c
diff --git a/auto_round/__main__.py b/auto_round/__main__.pyindex 7785b8125..6bcebcde0 100644--- a/auto_round/__main__.py+++ b/auto_round/__main__.py@@ -53,9 +53,6 @@ def run_lmms():lmms_eval(args)def switch():- # if "--lmms" in sys.argv:- # sys.argv.remove("--lmms")- # run_lmms()if "--mllm" in sys.argv:sys.argv.remove("--mllm")run_mllm()diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 1371dc20b..1bfc82883 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1263,6 +1263,9 @@ class AutoRound(object):self.model.save_pretrained(output_dir)if self.tokenizer is not None:self.tokenizer.save_pretrained(output_dir)+ processor = kwargs.get("processor", None)+ if processor is not None:+ processor.save_pretrained(output_dir)returnfrom auto_round.export import EXPORT_FORMATdiff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex fa07bd778..ccd1bc7a7 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -38,7 +38,7 @@ def _only_text_test(model, tokenizer, device):tokenizer.padding_side = 'left'if tokenizer.pad_token is None:tokenizer.pad_token = tokenizer.eos_token- if device != model.device.type:+ if device.split(':')[0] != model.device.type:model = model.to(device)inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True).to(model.device)model(**inputs)@@ -150,19 +150,20 @@ class AutoRoundMLLM(AutoRound):self.to_quant_block_names = to_quant_block_namesself.extra_data_dir = extra_data_dirself.quant_nontext_module = quant_nontext_module+ self.processor = processorself.image_processor = image_processorself.template = template if template is not None else model.config.model_typeif not isinstance(dataset, torch.utils.data.DataLoader):self.template = get_template(self.template, model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor)-- dataset = self.template.default_dataset if dataset is None else dataset+ dataset = self.template.default_dataset if dataset is None else datasetfrom ..calib_dataset import CALIB_DATASETSfrom .mllm_dataset import MLLM_DATASETif isinstance(dataset, str):if quant_nontext_module or \- (dataset in CALIB_DATASETS.keys() and not _only_text_test(model, tokenizer, device)):+ (dataset in CALIB_DATASETS.keys() and not \+ _only_text_test(model, tokenizer, device)):if quant_nontext_module:logger.warning(f"Text only dataset cannot be used for calibrating non-text modules,""switching to liuhaotian/llava_conv_58k")@@ -372,4 +373,20 @@ class AutoRoundMLLM(AutoRound):m = m.to("meta")# torch.cuda.empty_cache()+ def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):+ """Save the quantized model to the specified output directory in the specified format.++ Args:+ output_dir (str, optional): The directory to save the quantized model. Defaults to None.+ format (str, optional): The format in which to save the model. Defaults to "auto_round".+ inplace (bool, optional): Whether to modify the model in place. Defaults to True.+ **kwargs: Additional keyword arguments specific to the export format.+ Returns:+ object: The compressed model object.+ """+ if self.processor is not None and not hasattr(self.processor, "chat_template"):+ self.processor.chat_template = None+ compressed_model = super().save_quantized(+ output_dir=output_dir, format=format, inplace=inplace, processor=self.processor, **kwargs)+ return compressed_modeldiff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex 970fd1f87..08b4d9eb3 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -118,24 +118,25 @@ def _register_template(def load_template(path: str):"""Load template information from a json file."""- data = json.load(open(path, "r"))- if "model_type" not in data:- data["model_type"] = "user_define"- if "replace_tokens" in data and data["replace_tokens"] is not None:- assert len(data["replace_tokens"]) % 2 == 0, \- "the format of replace_tokens should be [old_tag1, replace_tag1, old_tag2, replace_tag2]"- temp = []- for i in range(0, len(data["replace_tokens"]), 2):- temp.append((data["replace_tokens"][i], data["replace_tokens"][i + 1]))- data["replace_tokens"] = temp- if "processor" in data:- assert data["processor"] in PROCESSORS.keys(), \- "{} is not supported, current support: {}".format(data["processor"], ",".join(PROCESSORS.keys()))- data["processor"] = PROCESSORS[data["processor"]]- template = _register_template(- **data- )- return template+ with open(path, "r") as file:+ data = json.load(file)+ if "model_type" not in data:+ data["model_type"] = "user_define"+ if "replace_tokens" in data and data["replace_tokens"] is not None:+ assert len(data["replace_tokens"]) % 2 == 0, \+ "the format of replace_tokens should be [old_tag1, replace_tag1, old_tag2, replace_tag2]"+ temp = []+ for i in range(0, len(data["replace_tokens"]), 2):+ temp.append((data["replace_tokens"][i], data["replace_tokens"][i + 1]))+ data["replace_tokens"] = temp+ if "processor" in data:+ assert data["processor"] in PROCESSORS.keys(), \+ "{} is not supported, current support: {}".format(data["processor"], ",".join(PROCESSORS.keys()))+ data["processor"] = PROCESSORS[data["processor"]]+ template = _register_template(+ **data+ )+ return templatedef _load_preset_template():diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 3c88a5b49..89e5b23a3 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -418,13 +418,11 @@ def tune(args):inplace = False if len(format_list) > 1 else Truefor format_ in format_list:eval_folder = f'{export_dir}-{format_}'- if processor is not None and not hasattr(processor, "chat_template"):- processor.chat_template = Nonesafe_serialization = Trueif "phi3_v" in model_type:safe_serialization = Falseautoround.save_quantized(- eval_folder, format=format_, inplace=inplace, processor=processor, safe_serialization=safe_serialization)+ eval_folder, format=format_, inplace=inplace, safe_serialization=safe_serialization)def eval(args):