PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support for more vlms (#390)
Heng Guo9c7a443d125c
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 5a4e64689..4260e6720 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -54,7 +54,9 @@ from .utils import (find_matching_blocks, is_debug_mode,TORCH_VERSION_AT_LEAST_2_6,supported_layer_types,- get_layer_features, set_module,+ get_layer_features,+ set_module,+ llm_load_model)from .low_cpu_mem.utils import get_layers_before_block@@ -165,8 +167,14 @@ class AutoRound(object):device_map: Union[str, dict] = None,super_bits: int = None,super_group_size: int = None,+ model_kwargs: dict = None,**kwargs,):+ if isinstance(model, str):+ torch_dtype = "auto"+ if device is not None and "hpu" in device:+ torch_dtype = torch.bfloat16+ model, tokenizer, low_cpu_mem_usage = llm_load_model(model, torch_dtype=torch_dtype, **model_kwargs)self.quantized = Falseself.model_orig_dtype = model.dtypeself.seed = seeddiff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex a481290ca..102dcae3d 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -22,8 +22,9 @@ import torch.nn as nnimport transformersimport auto_round.export.export_to_autoround.qlinear_triton_act+import auto_round_extension.cuda.qlinear_tritonv2-from auto_round.utils import get_module, logger, set_module, supported_layer_types+from auto_round.utils import get_module, logger, set_module, supported_layer_types, check_to_quantizedimport threadpoolctl as tctlimport inspectfrom tqdm import tqdm@@ -159,7 +160,7 @@ def pack_layer(layer_name, model, backend):if int(layer.act_bits) <= 8:return pack_qact_layer(layer_name, model)- if int(layer.bits) > 8:+ if not check_to_quantized(layer):returndevice = layer.weight.devicediff --git a/auto_round/mllm/README.md b/auto_round/mllm/README.mdindex bb58b8e2d..0bb1ab020 100644--- a/auto_round/mllm/README.md+++ b/auto_round/mllm/README.md@@ -81,13 +81,20 @@ quantization of non-text modules.Currently, the quantization of vision components is supported for Llama-3.2-11B-Vision, Phi-3.5-Vision-Instruct, andLlava-v1.5-7B.-| Model | Eval Lib | calibration dataset | quant nontext module |-|--------------|-----------|---------------------|----------------------|-| Qwen2-VL | vlmeval | pile/llava | - |-| Llama-Vision | lmms_eval | llava | ✔ |-| Phi3-Vision | vlmeval | pile/llava | ✔ |-| Llava-v1.5 | lmms_eval | pile/llava | - |-| CogVLM2 | lmms_eval | pile/llava | ✔ |+Support Matrix+| Model | calibration dataset | quant nontext module |+|-----------------------|---------------------|----------------------|+| Qwen2-VL | pile/llava | - |+| Llama-3.2-Vision | llava | ✔ |+| Phi3-Vision | pile/llava | ✔ |+| Llava-v1.5 | pile/llava | X |+| CogVLM2 | pile/llava | ✔ |+| gemma-3 | pile/llava | - |+| granite-vision-3.2 | pile/llava | - |+| Mistral-Small-3.1 | pile/llava | X |+| Aria | pile/llava | - |++✔ means support, - means support but cannot infer or not test infert yet, X means not support.### New Models Supportdiff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex e0ba63586..1bf78727e 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -26,11 +26,12 @@ from ..utils import (get_block_names,find_matching_blocks,extract_block_names_to_str,- clear_memory+ clear_memory,+ mllm_load_model)from ..autoround import AutoRoundfrom .template import get_template, Template-from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS+from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS, _handle_special_modelfrom .mllm_dataset import get_mllm_dataloaderfrom ..low_cpu_mem.utils import get_layers_before_block@@ -121,7 +122,7 @@ class AutoRoundMLLM(AutoRound):def __init__(self,model,- tokenizer,+ tokenizer = None,processor = None,image_processor = None,bits: int = 4,@@ -161,8 +162,17 @@ class AutoRoundMLLM(AutoRound):enable_norm_bias_tuning: bool = False,truncation: bool = None,enable_torch_compile: bool = False,+ model_kwargs: dict = None,**kwargs,):+ if isinstance(model, str):+ torch_dtype = "auto"+ if device is not None and "hpu" in device:+ torch_dtype = torch.bfloat16+ model, processor, tokenizer, image_processor = mllm_load_model(+ model, torch_dtype=torch_dtype, **model_kwargs)++ assert tokenizer is not None, "tokenizer should not be None."all_blocks = get_block_names(model, quant_nontext_module)self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)if to_quant_block_names is None:@@ -177,7 +187,9 @@ class AutoRoundMLLM(AutoRound):self.template = get_template(self.template, model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor)dataset = self.template.default_dataset if dataset is None else dataset-++ model = _handle_special_model(model)+from ..calib_dataset import CALIB_DATASETSfrom .mllm_dataset import MLLM_DATASETif isinstance(dataset, str):@@ -276,6 +288,7 @@ class AutoRoundMLLM(AutoRound):template=self.template,model=self.model,tokenizer=self.tokenizer,+ processor=self.processor,image_processor=self.image_processor,dataset=dataset,extra_data_dir=self.extra_data_dir,@@ -333,7 +346,8 @@ class AutoRoundMLLM(AutoRound):)data_new = {}for key in data.keys():- data_new[key] = to_device(data[key], self.model.device)+ data_new[key] = torch.tensor(data[key])+ data_new[key] = to_device(data_new[key], self.model.device)if key == 'images':data_new[key] = to_dtype(data_new[key], self.model.dtype)input_ids = data_new["input_ids"]@@ -344,9 +358,12 @@ class AutoRoundMLLM(AutoRound):data_new = {}for key in data.keys():data_new[key] = to_device(data[key], self.model.device)- if key == 'images':+ if key in ['images', 'pixel_values']:data_new[key] = to_dtype(data_new[key], self.model.dtype)- input_ids = data_new["input_ids"]+ if "input_ids" in data_new:+ input_ids = data_new["input_ids"]+ else:+ input_ids = data_new["inputs_embeds"]if input_ids.shape[-1] < self.seqlen:pbar.update(1)@@ -410,4 +427,3 @@ class AutoRoundMLLM(AutoRound):compressed_model = super().save_quantized(output_dir=output_dir, format=format, inplace=inplace, processor=self.processor, **kwargs)return compressed_model-diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.pyindex 400f5773e..18b7d0fd5 100644--- a/auto_round/mllm/mllm_dataset.py+++ b/auto_round/mllm/mllm_dataset.py@@ -190,6 +190,7 @@ def get_mllm_dataloader(template,model,tokenizer,+ processor,image_processor=None,dataset="liuhaotian/llava_conv_58k",extra_data_dir=None,@@ -222,7 +223,9 @@ def get_mllm_dataloader("""if isinstance(template, str):from .template import get_template- template = get_template(template, model=model, tokenizer=tokenizer, image_processor=image_processor)+ template = get_template(+ template, model=model, tokenizer=tokenizer,+ processor=processor, image_processor=image_processor)if os.path.isfile(dataset) or dataset in MLLM_DATASET.keys():dataset = MLLM_DATASET['liuhaotian/llava'](diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.pyindex 2ec127622..06a4102b7 100644--- a/auto_round/mllm/processor.py+++ b/auto_round/mllm/processor.py@@ -11,7 +11,22 @@# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.-+"""+Support Matrix+| Model | calibration dataset | quant nontext module |+|-----------------------|---------------------|----------------------|+| Qwen2-VL | pile/llava | - |+| Llama-3.2-Vision | llava | ✔ |+| Phi3-Vision | pile/llava | ✔ |+| Llava-v1.5 | pile/llava | X |+| CogVLM2 | pile/llava | ✔ |+| gemma-3 | pile/llava | - |+| granite-vision-3.2 | pile/llava | - |+| Mistral-Small-3.1 | pile/llava | X |+| Aria | pile/llava | - |++✔ means support, - means support but cannot infer or not test infert yet, X means not support.+"""import torchfrom transformers.data.data_collator import default_data_collator@@ -32,7 +47,7 @@ def register_processor(name):class BasicProcessor:def __init__(self):pass-+def post_init(self, model, tokenizer, processor=None, image_processor=None, **kwargs):self.model = modelself.tokenizer = tokenizer@@ -46,55 +61,9 @@ class BasicProcessor:self,text,images,- return_tensors="pt",squeeze=True,- max_length=None,- truncation=False,- truncation_strategy="text",**kwargs):-- if isinstance(text, list):- for message in text:- if not ("role" in message and "content" in message):- raise ValueError(- "When passing chat dicts as input, each dict must have a 'role' and 'content' key.")- continue_final_message = text[-1]["role"] == "assistant"-- # add_generation_prompt=True will add <|im_start|>assistant to the end- try:- text = self.tokenizer.apply_chat_template(- text, tokenize=False, add_generation_prompt=not continue_final_message,- continue_final_message=continue_final_message, )- except:- raise NotImplementedError("current not support for non-instructed model.")-- if text == '':- raise NotImplementedError("current not support for non-instructed model.")-- if images is not None:- images = self.image_processor(images)-- if truncation is True and truncation_strategy == "text":- text = self.tokenizer.decode(self.tokenizer(text).input_ids[:max_length])-- ret = self.processor(- text=text,- images=images,- return_tensors=return_tensors,- # videos = None- )- if truncation is True and truncation_strategy == "token":- seqlen = ret['input_ids'].shape[-1]- for key in ret:- shape_ = ret[key].shape- if len(shape_) == 2 and shape_[-1] == seqlen:- ret[key] = ret[key][:, :max_length]- elif len(shape_) == 4 and shape_[1] == seqlen:- ret[key] = ret[key][:, :max_length]-- if squeeze:- ret = self.squeeze_result(ret)- return ret+ raise NotImplementedError@staticmethoddef data_collator(batch):@@ -111,8 +80,102 @@ class BasicProcessor:return ret+@register_processor("hf")+class HFProcessor(BasicProcessor):+ # evaluation on: Qwen2-VL, mllama, Mistral-Small+ IMAGE_TOKEN = '<image>'+ def __init__(self):+ self.process_func = self._process_v1++ def post_init(self, model, tokenizer, processor=None, image_processor=None, **kwargs):+ self.model = model+ self.tokenizer = tokenizer+ self.processor = processor+ if image_processor is not None:+ self.image_processor = image_processor+ else:+ self.image_processor = self.default_image_processor++ def _process_v1(self, messages, image):+ """support models: Qwen2-VL, gemma-3, granite-vision-3.2, Aria"""+ conversation = []+ for content in messages:+ conversation.append({+ "role": content['role'],+ "content": [+ {"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}+ ]+ })+ if self.IMAGE_TOKEN in content['content']:+ conversation[-1]["content"].append({"image": image, "type": "image"})+ ret = self.processor.apply_chat_template(+ conversation, add_generation_prompt=True, tokenize=True, return_dict=True)+ return ret++ def _process_v2(self, messages, image):+ """support model: Mistral-Small-3.1, phi3_v"""+ conversation = []+ for content in messages:+ if content["role"] == "user":+ conversation.append({+ "role": content['role'],+ "content": [+ {"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}+ ]+ })+ if self.IMAGE_TOKEN in content['content']:+ conversation[-1]["content"].append({"image": image, "type": "image"})+ else:+ conversation.append({+ "role": content['role'],+ "content": content["content"]+ })+ if hasattr(self.processor, "chat_template"):+ text = self.processor.apply_chat_template(+ conversation, add_generation_prompt=True, tokenize=False, return_dict=False)+ else:+ continue_final_message = messages[-1]["role"] == "assistant"+ text = self.tokenizer.apply_chat_template(+ messages, tokenize=False, add_generation_prompt=not continue_final_message,+ continue_final_message=continue_final_message, )+ if image is not None:+ image = self.image_processor(image)+ ret = self.processor(+ text=text, images=image, return_tensors="pt")+ return ret++ def get_input(+ self,+ text,+ images,+ return_tensors="pt",+ squeeze=True,+ max_length=None,+ truncation=False,+ truncation_strategy="text",+ **kwargs):++ if isinstance(text, list):+ try:+ ret = self.process_func(text, images)+ except Exception:+ self.process_func = self._process_v2+ ret = self.process_func(text, images)+ else:+ text = self.tokenizer.decode(self.tokenizer(text).input_ids[:max_length])++ if images is not None:+ images = self.image_processor(images)+ ret = self.processor(+ text=text, images=images, return_tensors="pt", add_special_tokens=False)++ if squeeze:+ ret = self.squeeze_result(ret)+ return ret++@register_processor("qwen2_vl")-class Qwen2VLProcessor(BasicProcessor):+class Qwen2VLProcessor(HFProcessor):@staticmethoddef squeeze_result(ret):for key in ret:@@ -130,7 +193,7 @@ class CogVLM2Processor(BasicProcessor):if images is not None:images = self.image_processor(images)-+padding_len = 2303max_length = 0 if max_length is None else max_lengthmax_length += padding_len@@ -246,3 +309,40 @@ class LlavaProcessor(BasicProcessor):def data_collator(self, batch):return self.collator_func(batch)++@register_processor("deepseek_v2")+class DeepSeekV2Processor(BasicProcessor):+ IMAGE_TOKEN = '<image>'+ def get_input(+ self, text, images, max_length=None,+ squeeze=True, truncation=False, truncation_strategy="text", **kwargs):++ messages = []+ for content in text:+ if content["role"] == "user":+ messages.append({+ "role": content['role'],+ "content": content["content"]+ })+ if self.IMAGE_TOKEN in content['content']:+ messages[-1]["images"] = [images]+ else:+ messages.append({+ "role": content['role'],+ "content": content["content"]+ })++ if images is not None:+ pil_image = [self.image_processor(images)]+ else:+ pil_image = None++ prepare_inputs = self.processor(+ conversations=messages,+ images=pil_image,+ force_batchify=True,+ system_prompt=""+ ).to(self.model.device)+ prepare_inputs = prepare_inputs.to(self.model.device)+ prepare_inputs = self.squeeze_result(dict(prepare_inputs))+ return prepare_inputsdiff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.pyindex 7636206a8..62e1a2225 100644--- a/auto_round/mllm/template.py+++ b/auto_round/mllm/template.py@@ -115,6 +115,10 @@ def _register_template()return TEMPLATES[model_type]+_register_template("qwen2_vl", default_dataset="NeelNanda/pile-10k",processor=PROCESSORS["qwen2_vl"])+_register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCESSORS["hf"])+_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k",processor=PROCESSORS["deepseek_v2"])+_register_template("mistral3", default_dataset="NeelNanda/pile-10k",processor=PROCESSORS["hf"])def load_template(path: str):"""Load template information from a json file."""diff --git a/auto_round/mllm/templates/default.json b/auto_round/mllm/templates/default.jsonindex 321a0c485..a64017c73 100644--- a/auto_round/mllm/templates/default.json+++ b/auto_round/mllm/templates/default.json@@ -10,5 +10,5 @@"replace_tokens": null,"extra_encode" : false,"default_dataset": "NeelNanda/pile-10k",- "processor": "basic"+ "processor": "hf"}\ No newline at end of filediff --git a/auto_round/mllm/templates/mllama.json b/auto_round/mllm/templates/mllama.jsondeleted file mode 100644index 2434dc4ba..000000000--- a/auto_round/mllm/templates/mllama.json+++ /dev/null@@ -1,10 +0,0 @@-{- "model_type": "mllama",- "format_user": "<|start_header_id|>user<|end_header_id|>\n{{content}}<|eot_id|>",- "format_assistant": "<|start_header_id|>assistant<|end_header_id|>\n{{content}}<|eot_id|>",- "format_system": "<|begin_of_text|><|start_header_id|>system|end_header_id|>\n{{content}}<<eot_id>",- "default_system": "You are a helpful assistant.",- "replace_tokens": ["<image>", "<|image|>"],- "extra_encode" : true,- "default_dataset": "liuhaotian/llava_conv_58k"-}\ No newline at end of filediff --git a/auto_round/mllm/templates/phi3_v.json b/auto_round/mllm/templates/phi3_v.jsonindex 65e7445dc..212e00d16 100644--- a/auto_round/mllm/templates/phi3_v.json+++ b/auto_round/mllm/templates/phi3_v.json@@ -1,7 +1,7 @@{"model_type": "phi3_v","replace_tokens": ["<image>", "<|image_1|>"],- "processor": "basic",+ "processor": "hf","extra_encode" : false,"default_dataset": "NeelNanda/pile-10k"}\ No newline at end of filediff --git a/auto_round/mllm/templates/qwen2_vl.json b/auto_round/mllm/templates/qwen2_vl.jsondeleted file mode 100644index 6b72a0637..000000000--- a/auto_round/mllm/templates/qwen2_vl.json+++ /dev/null@@ -1,13 +0,0 @@-{- "model_type": "qwen2_vl",- "format_user": "<|im_start|>user\n{{content}}<|im_end|>\n",- "format_assistant": "<|im_start|>assistant\n{{content}}<|im_end|>\n",- "format_system": "<|im_start|>system\n{{content}}<|im_end|>\n",- "format_observation": "<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n",- "format_separator": "\n",- "default_system": "You are a helpful assistant.",- "replace_tokens": ["<image>", "<|vision_start|><|image_pad|><|vision_end|>"],- "processor": "qwen2_vl",- "extra_encode" : true,- "default_dataset": "NeelNanda/pile-10k"-}\ No newline at end of filediff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 6b221845a..e0a5411ac 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -369,66 +369,20 @@ def tune(args):if device_str is not None and "hpu" in device_str:torch_dtype = torch.bfloat16- is_glm = bool(re.search("chatglm", model_name.lower()))- low_cpu_mem_usage = False-- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)-- model_cls = AutoModel if is_glm else AutoModelForCausalLM-- if args.low_cpu_mem_tmp_dir is None:- args.low_cpu_mem_tmp_dir = os.path.join(args.output_dir, "low_cpu_mem_tmp")- if args.low_cpu_mem_mode == 2:- from auto_round.low_cpu_mem.utils import load_model_with_hooks- model = load_model_with_hooks(- model_name,- model_cls,- device=device_str,- clean_weight=True,- saved_path=args.low_cpu_mem_tmp_dir,- torch_dtype=torch_dtype,- trust_remote_code=not args.disable_trust_remote_code)- elif args.low_cpu_mem_mode == 1:- from auto_round.low_cpu_mem.utils import load_empty_model- low_cpu_mem_usage = True- model = load_empty_model(- model_name,- model_cls,- device=device_str,- saved_path=args.low_cpu_mem_tmp_dir,- torch_dtype=torch_dtype,- trust_remote_code=not args.disable_trust_remote_code)- else:- from auto_round.utils import _use_hpu_compile_mode- if _use_hpu_compile_mode():- model = model_cls.from_pretrained(- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype,- attn_implementation="eager",- trust_remote_code=not args.disable_trust_remote_code, device_map="auto" if use_auto_mapping else None- )- else:- model = model_cls.from_pretrained(- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype,- trust_remote_code=not args.disable_trust_remote_code, device_map="auto" if use_auto_mapping else None- )+ from auto_round.utils import llm_load_model+ model, tokenizer, low_cpu_mem_usage = llm_load_model(+ model_name,+ torch_dtype=torch_dtype,+ use_auto_mapping=use_auto_mapping,+ trust_remote_code=not args.disable_trust_remote_code,+ device=device_str,+ low_cpu_mem_mode=args.low_cpu_mem_mode,+ low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir)from auto_round import AutoRound, AutoRoundAdam- model = model.eval()seqlen = args.seqlen- if args.model_dtype != None:- try:- if args.model_dtype == "float16" or args.model_dtype == "fp16":- model = model.to(torch.float16)- elif args.model_dtype == "bfloat16" or args.model_dtype == "bfp16" or args.model_dtype == "bf16":- model = model.to(torch.bfloat16)- elif args.model_dtype == "float32" or args.model_dtype == "fp32":- model = model.to(torch.float32)- except:- logger.error("please use more device to fit the device or just use one device")- exit()-if hasattr(tokenizer, "model_max_length"):if tokenizer.model_max_length < seqlen:logger.info(diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 4c74a1732..6d06aca1c 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -322,70 +322,17 @@ def tune(args):torch_dtype = torch.bfloat16# load_model- processor, image_processor = None, None- if "deepseek-vl2" in model_name.lower():- from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM # pylint: disable=E0401- processor = DeepseekVLV2Processor.from_pretrained(model_name)- tokenizer = processor.tokenizer- model: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(- model_name,- trust_remote_code=not args.disable_trust_remote_code,- torch_dtype=torch_dtype,- device_map="auto" if use_auto_mapping else None)- model_type = "deepseek_vl_v2"- else:- config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)- if "llava" in model_name and config.architectures[0] != "LlavaForConditionalGeneration":- from llava.model.builder import load_pretrained_model # pylint: disable=E0401- tokenizer, model, image_processor, _ = load_pretrained_model(- model_name, model_base=None, model_name=model_name, torch_dtype=torch_dtype)- model_type = "llava"- else:- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)- model_type = config.model_type- if "llava" in model_type:- from transformers import LlavaForConditionalGeneration- cls = LlavaForConditionalGeneration- elif "qwen2_vl" in model_type:- from transformers import Qwen2VLForConditionalGeneration- cls = Qwen2VLForConditionalGeneration- elif "mllama" in model_type:- from transformers import MllamaForConditionalGeneration- cls = MllamaForConditionalGeneration- elif "gemma3" in model_type:- from transformers import Gemma3ForConditionalGeneration- cls = Gemma3ForConditionalGeneration- elif "mistral3" in model_type:- from transformers import Mistral3ForConditionalGeneration- cls = Mistral3ForConditionalGeneration- else:- cls = AutoModelForCausalLM-- model = cls.from_pretrained(- model_name,- trust_remote_code=not args.disable_trust_remote_code,- torch_dtype=torch_dtype,- device_map="auto" if use_auto_mapping else None)- if "cogvlm2" in model_name:- model.config.model_type = "cogvlm2"+ from auto_round.utils import mllm_load_model+ model, processor, tokenizer, image_processor = mllm_load_model(+ model_name,+ torch_dtype=torch_dtype,+ use_auto_mapping=use_auto_mapping,+ trust_remote_code=not args.disable_trust_remote_code)from auto_round import AutoRoundMLLMmodel = model.eval()- if args.model_dtype != None:- try:- if args.model_dtype == "float16" or args.model_dtype == "fp16":- model = model.to(torch.float16)- elif args.model_dtype == "bfloat16" or args.model_dtype == "bfp16" or args.model_dtype == "bf16":- model = model.to(torch.bfloat16)- elif args.model_dtype == "float32" or args.model_dtype == "fp32":- model = model.to(torch.float32)- except:- logger.error("please use more device to fit the device or just use one device")- exit()-round = AutoRoundMLLMlayer_config = {}@@ -399,7 +346,6 @@ def tune(args):##TODO gptq could support some mixed precision configlogger.warning(f"mixed precision exporting does not support {format} currently")- layer_config = {}if args.fp_layers != "":fp_layers = args.fp_layers.replace(" ", "").split(",")for n, m in model.named_modules():@@ -461,6 +407,11 @@ def tune(args):enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False+ model_kwargs = {+ "use_auto_mapping": use_auto_mapping,+ "trust_remote_code": not args.disable_trust_remote_code,+ "model_dtype": args.model_dtype+ }autoround = round(model,tokenizer,@@ -494,7 +445,9 @@ def tune(args):not_use_best_mse=args.not_use_best_mse,to_quant_block_names=args.to_quant_block_names,enable_torch_compile=enable_torch_compile,- device_map=args.device_map)+ device_map=args.device_map,+ model_kwargs=model_kwargs+ )model, _ = autoround.quantize()model.eval()@@ -510,8 +463,9 @@ def tune(args):for format_ in format_list:eval_folder = f'{export_dir}-{format_}'safe_serialization = True- if "phi3_v" in model_type:- safe_serialization = False+ if hasattr(autoround.model, "config") and hasattr(autoround.model.config, "model_type"):+ if "phi3_v" in autoround.model.config.model_type:+ safe_serialization = Falseautoround.save_quantized(eval_folder, format=format_, inplace=inplace, safe_serialization=safe_serialization)diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 03aa019a7..de8eb4ea7 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -28,6 +28,14 @@ SUPPORT_ONLY_TEXT_MODELS = ["idefics3"]+def _handle_special_model(model):+ if model.config.model_type == "deepseek_vl_v2":+ from auto_round.special_model_handler import _deepseek_vl2_forward+ from functools import partial+ model.forward = partial(_deepseek_vl2_forward, model)+ return model++def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):model.forward = model.language.forwardblock_names = []@@ -41,6 +49,46 @@ SPECIAL_MULTIMODAL_BLOCK = {"deepseek_vl_v2": _get_deepseek_vl2_multimodal_block}+def _deepseek_vl2_forward(+ model,+ input_ids = None,++ position_ids = None,+ attention_mask = None,+ past_key_values = None,+ inputs_embeds = None,++ images = None,+ images_seq_mask = None,+ images_spatial_crop = None,++ labels = None,+ use_cache = None,+ output_attentions = None,+ output_hidden_states = None,+ return_dict = None,+ cache_position = None,+ **kwargs+ ):+ inputs_embeds = model.prepare_inputs_embeds(+ input_ids=input_ids,+ images=images,+ images_seq_mask=images_seq_mask,+ images_spatial_crop=images_spatial_crop,+ )+ return model.language(+ input_ids=None,+ attention_mask=attention_mask,+ position_ids=position_ids,+ past_key_values=past_key_values,+ inputs_embeds=inputs_embeds,+ labels=labels,+ use_cache=use_cache,+ output_attentions=output_attentions,+ output_hidden_states=output_hidden_states,+ return_dict=return_dict,+ cache_position=cache_position)+def to_device(input, device=torch.device("cpu")):"""Moves input data to the specified device.@@ -103,7 +151,7 @@ def reset_params(inputs):"""if "use_cache" in inputs.keys(): # Not storing kv cacheinputs['use_cache'] = False-+def check_skippable_keywords(key):"""@@ -113,7 +161,7 @@ def check_skippable_keywords(key):if cache_key not in key:return Truereturn False-+def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):"""@@ -126,4 +174,3 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")return 1, accumulate_stepsreturn batch_size, gradient_accumulate_steps-diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 9e1b64511..ea9779893 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -421,7 +421,7 @@ def get_block_names(model, quant_vision=False):for n, m in target_m[1].named_children():block_names[i].append(target_m[0] + "." + n)return block_names-+if quant_vision or not is_pure_text_model(model):return _get_vlm_block_names(model, quant_vision=quant_vision)else:@@ -1229,7 +1229,7 @@ def _gguf_args_check(args):sys.exit(-1)else:pre_dq_format = format-+if os.path.isdir(args.model):from pathlib import Pathfrom auto_round.export.export_to_gguf.convert import Model@@ -1265,6 +1265,144 @@ def _gguf_args_check(args):return args++def _to_model_dtype(model, model_dtype):+ if model_dtype != None:+ try:+ if model_dtype == "float16" or model_dtype == "fp16":+ model = model.to(torch.float16)+ elif model_dtype == "bfloat16" or model_dtype == "bfp16" or model_dtype == "bf16":+ model = model.to(torch.bfloat16)+ elif model_dtype == "float32" or model_dtype == "fp32":+ model = model.to(torch.float32)+ except:+ logger.error("please use more device to fit the device or just use one device")+ exit()+ return model++def llm_load_model(+ pretrained_model_name_or_path,+ torch_dtype="auto",+ use_auto_mapping=True,+ trust_remote_code=True,+ model_dtype=None,+ device="cpu",+ low_cpu_mem_mode=0,+ low_cpu_mem_tmp_dir = None,+ **kwargs):+ from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM++ is_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))+ low_cpu_mem_usage = False++ tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)++ model_cls = AutoModel if is_glm else AutoModelForCausalLM++ if low_cpu_mem_tmp_dir is None:+ low_cpu_mem_tmp_dir = "low_cpu_mem_tmp"+ if low_cpu_mem_mode == 2:+ from auto_round.low_cpu_mem.utils import load_model_with_hooks+ model = load_model_with_hooks(+ pretrained_model_name_or_path,+ model_cls,+ device=device,+ clean_weight=True,+ saved_path=low_cpu_mem_tmp_dir,+ torch_dtype=torch_dtype,+ trust_remote_code=trust_remote_code)+ elif low_cpu_mem_mode == 1:+ from auto_round.low_cpu_mem.utils import load_empty_model+ low_cpu_mem_usage = True+ model = load_empty_model(+ pretrained_model_name_or_path,+ model_cls,+ device=device,+ saved_path=low_cpu_mem_tmp_dir,+ torch_dtype=torch_dtype,+ trust_remote_code=trust_remote_code)+ else:+ from auto_round.utils import _use_hpu_compile_mode+ if _use_hpu_compile_mode():+ model = model_cls.from_pretrained(+ pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,+ attn_implementation="eager",+ trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None+ )+ else:+ model = model_cls.from_pretrained(+ pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,+ trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None+ )++ model = model.eval()+ model = _to_model_dtype(model, model_dtype)++ return model, tokenizer, low_cpu_mem_usage+++def mllm_load_model(+ pretrained_model_name_or_path,+ torch_dtype="auto",+ use_auto_mapping=True,+ trust_remote_code=True,+ model_dtype=None,+ **kwargs):+ import json+ import transformers+ from transformers import AutoProcessor, AutoTokenizer, AutoModelForCausalLM+ from huggingface_hub import HfApi, HfFileSystem++ if os.path.isdir(pretrained_model_name_or_path):+ config = json.load(open(os.path.join(pretrained_model_name_or_path, "config.json")))+ else:+ hf_file = HfFileSystem()+ config = json.load(hf_file.open(pretrained_model_name_or_path + "/config.json"))++ if "model_type" in config:+ model_type = config["model_type"]+ else:+ model_type = None++ processor, image_processor = None, None+ if "deepseek_vl_v2" == model_type:+ from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM # pylint: disable=E0401+ processor = DeepseekVLV2Processor.from_pretrained(pretrained_model_name_or_path)+ tokenizer = processor.tokenizer+ model: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(+ pretrained_model_name_or_path,+ trust_remote_code=trust_remote_code,+ torch_dtype=torch_dtype,+ device_map="auto" if use_auto_mapping else None)+ else:+ architectures = config["architectures"][0]+ if architectures == "LlavaLlamaForCausalLM":+ from llava.model.builder import load_pretrained_model # pylint: disable=E0401+ tokenizer, model, image_processor, _ = load_pretrained_model(+ pretrained_model_name_or_path,+ model_base=None,+ model_name=pretrained_model_name_or_path,+ torch_dtype=torch_dtype)+ else:+ if hasattr(transformers, architectures):+ cls = getattr(transformers, architectures)+ else:+ cls = AutoModelForCausalLM+ model = cls.from_pretrained(+ pretrained_model_name_or_path,+ trust_remote_code=trust_remote_code,+ torch_dtype=torch_dtype,+ device_map="auto" if use_auto_mapping else None)+ tokenizer = AutoTokenizer.from_pretrained(+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code)+ processor = AutoProcessor.from_pretrained(+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code)++ model = model.eval()+ model = _to_model_dtype(model, model_dtype)++ return model, processor, tokenizer, image_processor+def is_pure_text_model(model):"""verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl, """if hasattr(model,"config") and hasattr(model.config, "vision_config"):@@ -1280,4 +1418,4 @@ def is_pure_text_model(model):return Falseif "img" in str(module.__class__).lower():return False- return True+ return Truediff --git a/test/test_block_names.py b/test/test_block_names.pyindex c98885e50..b828a1a3a 100644--- a/test/test_block_names.py+++ b/test/test_block_names.py@@ -174,6 +174,19 @@ class TestQuantizationBlocks(unittest.TestCase):assert quant_config.to_quant_block_names is not Noneshutil.rmtree("./saved", ignore_errors=True)++ def test_mm_block_name(self):+ from auto_round.utils import get_block_names+ from transformers import Qwen2VLForConditionalGeneration+ model = Qwen2VLForConditionalGeneration.from_pretrained(+ self.model_name, trust_remote_code=True, device_map="auto")+ block_name = get_block_names(model, quant_vision=True)+ self.assertTrue(len(block_name) == 2)+ self.assertTrue(all(["visual.merger.mlp" not in n for n in block_name]))+ block_name = get_block_names(model, quant_vision=False)+ self.assertTrue(len(block_name) == 1)+ self.assertTrue(block_name == get_block_names(model))+def test_moe(self):from auto_round.utils import get_block_namesmodel_name = "Qwen/Qwen1.5-MoE-A2.7B"@@ -205,6 +218,7 @@ class TestQuantizationBlocks(unittest.TestCase):# quant_config = model.config.quantization_config# assert quant_config.to_quant_block_names is not None+if __name__ == "__main__":diff --git a/test/test_mllm.py b/test/test_mllm.pyindex 68b27752e..da811622c 100644--- a/test/test_mllm.py+++ b/test/test_mllm.py@@ -15,9 +15,13 @@ class FakeDataLoader:self.batch_size = 1self.data = {- "text": [{'role': 'user',- 'content': '<|vision_start|><|image_pad|><|vision_end|>\nWhat are the colors of the bus in the image?'},- {'role': 'assistant', 'content': 'The bus in the image is white and red.'}],+ "text": [{+ 'role': 'user',+ 'content': '<image>\nWhat are the colors of the bus in the image?'+ }, {+ 'role': 'assistant',+ 'content': 'The bus in the image is white and red.'+ }],"image": "http://images.cocodataset.org/train2017/000000033471.jpg"}@@ -46,10 +50,10 @@ class TestAutoRoundMLLM(unittest.TestCase):self.model_name, trust_remote_code=True, device_map="auto")bits, group_size = 4, 128autoround = AutoRoundMLLM(- model, tokenizer, processor=processor,+ model, tokenizer, processor=processor,bits=bits, group_size=group_size,nsamples=1,- batch_size=1, iters=2, dataset=self.dataset,seqlen=256)+ batch_size=1, iters=2, dataset=self.dataset,seqlen=10)autoround.quantize()autoround.save_quantized("./saved/", format="auto_gptq", inplace=False)autoround.save_quantized("./saved/", format="auto_round", inplace=False)@@ -64,10 +68,10 @@ class TestAutoRoundMLLM(unittest.TestCase):model, tokenizer, processor=processor,bits=bits, group_size=group_size,nsamples=5,- batch_size=3, iters=2, dataset=self.dataset, quant_nontext_module=True, seqlen=256)+ batch_size=3, iters=2, dataset=self.dataset, quant_nontext_module=True, seqlen=10)autoround.quantize()autoround.save_quantized("./saved/", format="auto_round", inplace=True)-+def test_quant_block_names(self):from auto_round.utils import get_block_names,find_matching_blockstokenizer = AutoTokenizer.from_pretrained(self.model_name)@@ -79,7 +83,7 @@ class TestAutoRoundMLLM(unittest.TestCase):all_blocks = get_block_names(model, quant_vision=True)blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)assert target_blocks == blocks-+def test_dataset_check(self):from auto_round.mllm.mllm_dataset import MLLM_DATASETclass Myclass:@@ -88,7 +92,7 @@ class TestAutoRoundMLLM(unittest.TestCase):self.assertEqual(len(dataset.questions), 32)dataset = MLLM_DATASET['liuhaotian/llava'](template=Myclass(), model=None, tokenzier=None, dataset_path="liuhaotian/llava", seqlen=2048, nsamples=512)self.assertEqual(len(dataset.questions), 512)-+def test_diff_dataset(self):tokenizer = AutoTokenizer.from_pretrained(self.model_name)processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)@@ -113,8 +117,6 @@ class TestAutoRoundMLLM(unittest.TestCase):self.assertTrue(is_pure_text_model(model))+if __name__ == "__main__":unittest.main()---diff --git a/test_cuda/test_vlms.py b/test_cuda/test_vlms.pyindex ab0438e0d..c32a061a8 100644--- a/test_cuda/test_vlms.py+++ b/test_cuda/test_vlms.py@@ -193,6 +193,34 @@ class TestAutoRound(unittest.TestCase):f"--quant_nontext_module --iters 1 --nsamples 1 --output_dir {absolute_path}")self.phi3_infernece(absolute_path)shutil.rmtree(absolute_path, ignore_errors=True)++ def test_mm_block_name(self):+ from auto_round.utils import get_block_names++ model_name = "/models/deepseek-vl2-tiny"+ from deepseek_vl2.models import DeepseekVLV2ForCausalLM+ model = DeepseekVLV2ForCausalLM.from_pretrained(+ model_name, trust_remote_code=True, device_map="auto")+ block_name = get_block_names(model, quant_vision=True)+ self.assertTrue(len(block_name) == 3)+ self.assertTrue(any(["projector.layers.1" not in n for n in block_name]))+ block_name = get_block_names(model, quant_vision=False)+ self.assertTrue(len(block_name) == 1)+ self.assertTrue(get_block_names(model) == block_name)++ model_name = "/models/Llama-3.2-11B-Vision-Instruct/"+ from transformers import MllamaForConditionalGeneration+ model = MllamaForConditionalGeneration.from_pretrained(+ model_name, trust_remote_code=True, device_map="auto")+ block_name = get_block_names(model, quant_vision=True)+ self.assertTrue(len(block_name) == 3)+ self.assertTrue(any(["vision_model.global_transformer.layers.0" not in n for n in block_name]))+ self.assertTrue(any(["vision_model.transformer.layers.0" not in n for n in block_name]))+ block_name = get_block_names(model, quant_vision=False)+ self.assertTrue(len(block_name) == 1)+ self.assertTrue(get_block_names(model) == block_name)++