Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support for more vlms (#390)

Heng Guo9c7a443d125c
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 5a4e64689..4260e6720 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -54,7 +54,9 @@ from .utils import (
find_matching_blocks, is_debug_mode,
TORCH_VERSION_AT_LEAST_2_6,
supported_layer_types,
- get_layer_features, set_module,
+ get_layer_features,
+ set_module,
+ llm_load_model
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -165,8 +167,14 @@ class AutoRound(object):
device_map: Union[str, dict] = None,
super_bits: int = None,
super_group_size: int = None,
+ model_kwargs: dict = None,
**kwargs,
):
+ if isinstance(model, str):
+ torch_dtype = "auto"
+ if device is not None and "hpu" in device:
+ torch_dtype = torch.bfloat16
+ model, tokenizer, low_cpu_mem_usage = llm_load_model(model, torch_dtype=torch_dtype, **model_kwargs)
self.quantized = False
self.model_orig_dtype = model.dtype
self.seed = seed
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index a481290ca..102dcae3d 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -22,8 +22,9 @@ import torch.nn as nn
import transformers
import auto_round.export.export_to_autoround.qlinear_triton_act
+
import auto_round_extension.cuda.qlinear_tritonv2
-from auto_round.utils import get_module, logger, set_module, supported_layer_types
+from auto_round.utils import get_module, logger, set_module, supported_layer_types, check_to_quantized
import threadpoolctl as tctl
import inspect
from tqdm import tqdm
@@ -159,7 +160,7 @@ def pack_layer(layer_name, model, backend):
if int(layer.act_bits) <= 8:
return pack_qact_layer(layer_name, model)
- if int(layer.bits) > 8:
+ if not check_to_quantized(layer):
return
device = layer.weight.device
diff --git a/auto_round/mllm/README.md b/auto_round/mllm/README.md
index bb58b8e2d..0bb1ab020 100644
--- a/auto_round/mllm/README.md
+++ b/auto_round/mllm/README.md
@@ -81,13 +81,20 @@ quantization of non-text modules.
Currently, the quantization of vision components is supported for Llama-3.2-11B-Vision, Phi-3.5-Vision-Instruct, and
Llava-v1.5-7B.
-| Model | Eval Lib | calibration dataset | quant nontext module |
-|--------------|-----------|---------------------|----------------------|
-| Qwen2-VL | vlmeval | pile/llava | - |
-| Llama-Vision | lmms_eval | llava | ✔ |
-| Phi3-Vision | vlmeval | pile/llava | ✔ |
-| Llava-v1.5 | lmms_eval | pile/llava | - |
-| CogVLM2 | lmms_eval | pile/llava | ✔ |
+Support Matrix
+| Model | calibration dataset | quant nontext module |
+|-----------------------|---------------------|----------------------|
+| Qwen2-VL | pile/llava | - |
+| Llama-3.2-Vision | llava | ✔ |
+| Phi3-Vision | pile/llava | ✔ |
+| Llava-v1.5 | pile/llava | X |
+| CogVLM2 | pile/llava | ✔ |
+| gemma-3 | pile/llava | - |
+| granite-vision-3.2 | pile/llava | - |
+| Mistral-Small-3.1 | pile/llava | X |
+| Aria | pile/llava | - |
+
+✔ means support, - means support but cannot infer or not test infert yet, X means not support.
### New Models Support
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index e0ba63586..1bf78727e 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -26,11 +26,12 @@ from ..utils import (
get_block_names,
find_matching_blocks,
extract_block_names_to_str,
- clear_memory
+ clear_memory,
+ mllm_load_model
)
from ..autoround import AutoRound
from .template import get_template, Template
-from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
+from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS, _handle_special_model
from .mllm_dataset import get_mllm_dataloader
from ..low_cpu_mem.utils import get_layers_before_block
@@ -121,7 +122,7 @@ class AutoRoundMLLM(AutoRound):
def __init__(
self,
model,
- tokenizer,
+ tokenizer = None,
processor = None,
image_processor = None,
bits: int = 4,
@@ -161,8 +162,17 @@ class AutoRoundMLLM(AutoRound):
enable_norm_bias_tuning: bool = False,
truncation: bool = None,
enable_torch_compile: bool = False,
+ model_kwargs: dict = None,
**kwargs,
):
+ if isinstance(model, str):
+ torch_dtype = "auto"
+ if device is not None and "hpu" in device:
+ torch_dtype = torch.bfloat16
+ model, processor, tokenizer, image_processor = mllm_load_model(
+ model, torch_dtype=torch_dtype, **model_kwargs)
+
+ assert tokenizer is not None, "tokenizer should not be None."
all_blocks = get_block_names(model, quant_nontext_module)
self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
if to_quant_block_names is None:
@@ -177,7 +187,9 @@ class AutoRoundMLLM(AutoRound):
self.template = get_template(
self.template, model=model, tokenizer=tokenizer, processor=processor, image_processor=image_processor)
dataset = self.template.default_dataset if dataset is None else dataset
-
+
+ model = _handle_special_model(model)
+
from ..calib_dataset import CALIB_DATASETS
from .mllm_dataset import MLLM_DATASET
if isinstance(dataset, str):
@@ -276,6 +288,7 @@ class AutoRoundMLLM(AutoRound):
template=self.template,
model=self.model,
tokenizer=self.tokenizer,
+ processor=self.processor,
image_processor=self.image_processor,
dataset=dataset,
extra_data_dir=self.extra_data_dir,
@@ -333,7 +346,8 @@ class AutoRoundMLLM(AutoRound):
)
data_new = {}
for key in data.keys():
- data_new[key] = to_device(data[key], self.model.device)
+ data_new[key] = torch.tensor(data[key])
+ data_new[key] = to_device(data_new[key], self.model.device)
if key == 'images':
data_new[key] = to_dtype(data_new[key], self.model.dtype)
input_ids = data_new["input_ids"]
@@ -344,9 +358,12 @@ class AutoRoundMLLM(AutoRound):
data_new = {}
for key in data.keys():
data_new[key] = to_device(data[key], self.model.device)
- if key == 'images':
+ if key in ['images', 'pixel_values']:
data_new[key] = to_dtype(data_new[key], self.model.dtype)
- input_ids = data_new["input_ids"]
+ if "input_ids" in data_new:
+ input_ids = data_new["input_ids"]
+ else:
+ input_ids = data_new["inputs_embeds"]
if input_ids.shape[-1] < self.seqlen:
pbar.update(1)
@@ -410,4 +427,3 @@ class AutoRoundMLLM(AutoRound):
compressed_model = super().save_quantized(
output_dir=output_dir, format=format, inplace=inplace, processor=self.processor, **kwargs)
return compressed_model
-
diff --git a/auto_round/mllm/mllm_dataset.py b/auto_round/mllm/mllm_dataset.py
index 400f5773e..18b7d0fd5 100644
--- a/auto_round/mllm/mllm_dataset.py
+++ b/auto_round/mllm/mllm_dataset.py
@@ -190,6 +190,7 @@ def get_mllm_dataloader(
template,
model,
tokenizer,
+ processor,
image_processor=None,
dataset="liuhaotian/llava_conv_58k",
extra_data_dir=None,
@@ -222,7 +223,9 @@ def get_mllm_dataloader(
"""
if isinstance(template, str):
from .template import get_template
- template = get_template(template, model=model, tokenizer=tokenizer, image_processor=image_processor)
+ template = get_template(
+ template, model=model, tokenizer=tokenizer,
+ processor=processor, image_processor=image_processor)
if os.path.isfile(dataset) or dataset in MLLM_DATASET.keys():
dataset = MLLM_DATASET['liuhaotian/llava'](
diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.py
index 2ec127622..06a4102b7 100644
--- a/auto_round/mllm/processor.py
+++ b/auto_round/mllm/processor.py
@@ -11,7 +11,22 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
+"""
+Support Matrix
+| Model | calibration dataset | quant nontext module |
+|-----------------------|---------------------|----------------------|
+| Qwen2-VL | pile/llava | - |
+| Llama-3.2-Vision | llava | ✔ |
+| Phi3-Vision | pile/llava | ✔ |
+| Llava-v1.5 | pile/llava | X |
+| CogVLM2 | pile/llava | ✔ |
+| gemma-3 | pile/llava | - |
+| granite-vision-3.2 | pile/llava | - |
+| Mistral-Small-3.1 | pile/llava | X |
+| Aria | pile/llava | - |
+
+✔ means support, - means support but cannot infer or not test infert yet, X means not support.
+"""
import torch
from transformers.data.data_collator import default_data_collator
@@ -32,7 +47,7 @@ def register_processor(name):
class BasicProcessor:
def __init__(self):
pass
-
+
def post_init(self, model, tokenizer, processor=None, image_processor=None, **kwargs):
self.model = model
self.tokenizer = tokenizer
@@ -46,55 +61,9 @@ class BasicProcessor:
self,
text,
images,
- return_tensors="pt",
squeeze=True,
- max_length=None,
- truncation=False,
- truncation_strategy="text",
**kwargs):
-
- if isinstance(text, list):
- for message in text:
- if not ("role" in message and "content" in message):
- raise ValueError(
- "When passing chat dicts as input, each dict must have a 'role' and 'content' key.")
- continue_final_message = text[-1]["role"] == "assistant"
-
- # add_generation_prompt=True will add <|im_start|>assistant to the end
- try:
- text = self.tokenizer.apply_chat_template(
- text, tokenize=False, add_generation_prompt=not continue_final_message,
- continue_final_message=continue_final_message, )
- except:
- raise NotImplementedError("current not support for non-instructed model.")
-
- if text == '':
- raise NotImplementedError("current not support for non-instructed model.")
-
- if images is not None:
- images = self.image_processor(images)
-
- if truncation is True and truncation_strategy == "text":
- text = self.tokenizer.decode(self.tokenizer(text).input_ids[:max_length])
-
- ret = self.processor(
- text=text,
- images=images,
- return_tensors=return_tensors,
- # videos = None
- )
- if truncation is True and truncation_strategy == "token":
- seqlen = ret['input_ids'].shape[-1]
- for key in ret:
- shape_ = ret[key].shape
- if len(shape_) == 2 and shape_[-1] == seqlen:
- ret[key] = ret[key][:, :max_length]
- elif len(shape_) == 4 and shape_[1] == seqlen:
- ret[key] = ret[key][:, :max_length]
-
- if squeeze:
- ret = self.squeeze_result(ret)
- return ret
+ raise NotImplementedError
@staticmethod
def data_collator(batch):
@@ -111,8 +80,102 @@ class BasicProcessor:
return ret
+@register_processor("hf")
+class HFProcessor(BasicProcessor):
+ # evaluation on: Qwen2-VL, mllama, Mistral-Small
+ IMAGE_TOKEN = '<image>'
+ def __init__(self):
+ self.process_func = self._process_v1
+
+ def post_init(self, model, tokenizer, processor=None, image_processor=None, **kwargs):
+ self.model = model
+ self.tokenizer = tokenizer
+ self.processor = processor
+ if image_processor is not None:
+ self.image_processor = image_processor
+ else:
+ self.image_processor = self.default_image_processor
+
+ def _process_v1(self, messages, image):
+ """support models: Qwen2-VL, gemma-3, granite-vision-3.2, Aria"""
+ conversation = []
+ for content in messages:
+ conversation.append({
+ "role": content['role'],
+ "content": [
+ {"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}
+ ]
+ })
+ if self.IMAGE_TOKEN in content['content']:
+ conversation[-1]["content"].append({"image": image, "type": "image"})
+ ret = self.processor.apply_chat_template(
+ conversation, add_generation_prompt=True, tokenize=True, return_dict=True)
+ return ret
+
+ def _process_v2(self, messages, image):
+ """support model: Mistral-Small-3.1, phi3_v"""
+ conversation = []
+ for content in messages:
+ if content["role"] == "user":
+ conversation.append({
+ "role": content['role'],
+ "content": [
+ {"text": content["content"].replace(self.IMAGE_TOKEN, ""), "type": "text"}
+ ]
+ })
+ if self.IMAGE_TOKEN in content['content']:
+ conversation[-1]["content"].append({"image": image, "type": "image"})
+ else:
+ conversation.append({
+ "role": content['role'],
+ "content": content["content"]
+ })
+ if hasattr(self.processor, "chat_template"):
+ text = self.processor.apply_chat_template(
+ conversation, add_generation_prompt=True, tokenize=False, return_dict=False)
+ else:
+ continue_final_message = messages[-1]["role"] == "assistant"
+ text = self.tokenizer.apply_chat_template(
+ messages, tokenize=False, add_generation_prompt=not continue_final_message,
+ continue_final_message=continue_final_message, )
+ if image is not None:
+ image = self.image_processor(image)
+ ret = self.processor(
+ text=text, images=image, return_tensors="pt")
+ return ret
+
+ def get_input(
+ self,
+ text,
+ images,
+ return_tensors="pt",
+ squeeze=True,
+ max_length=None,
+ truncation=False,
+ truncation_strategy="text",
+ **kwargs):
+
+ if isinstance(text, list):
+ try:
+ ret = self.process_func(text, images)
+ except Exception:
+ self.process_func = self._process_v2
+ ret = self.process_func(text, images)
+ else:
+ text = self.tokenizer.decode(self.tokenizer(text).input_ids[:max_length])
+
+ if images is not None:
+ images = self.image_processor(images)
+ ret = self.processor(
+ text=text, images=images, return_tensors="pt", add_special_tokens=False)
+
+ if squeeze:
+ ret = self.squeeze_result(ret)
+ return ret
+
+
@register_processor("qwen2_vl")
-class Qwen2VLProcessor(BasicProcessor):
+class Qwen2VLProcessor(HFProcessor):
@staticmethod
def squeeze_result(ret):
for key in ret:
@@ -130,7 +193,7 @@ class CogVLM2Processor(BasicProcessor):
if images is not None:
images = self.image_processor(images)
-
+
padding_len = 2303
max_length = 0 if max_length is None else max_length
max_length += padding_len
@@ -246,3 +309,40 @@ class LlavaProcessor(BasicProcessor):
def data_collator(self, batch):
return self.collator_func(batch)
+
+@register_processor("deepseek_v2")
+class DeepSeekV2Processor(BasicProcessor):
+ IMAGE_TOKEN = '<image>'
+ def get_input(
+ self, text, images, max_length=None,
+ squeeze=True, truncation=False, truncation_strategy="text", **kwargs):
+
+ messages = []
+ for content in text:
+ if content["role"] == "user":
+ messages.append({
+ "role": content['role'],
+ "content": content["content"]
+ })
+ if self.IMAGE_TOKEN in content['content']:
+ messages[-1]["images"] = [images]
+ else:
+ messages.append({
+ "role": content['role'],
+ "content": content["content"]
+ })
+
+ if images is not None:
+ pil_image = [self.image_processor(images)]
+ else:
+ pil_image = None
+
+ prepare_inputs = self.processor(
+ conversations=messages,
+ images=pil_image,
+ force_batchify=True,
+ system_prompt=""
+ ).to(self.model.device)
+ prepare_inputs = prepare_inputs.to(self.model.device)
+ prepare_inputs = self.squeeze_result(dict(prepare_inputs))
+ return prepare_inputs
diff --git a/auto_round/mllm/template.py b/auto_round/mllm/template.py
index 7636206a8..62e1a2225 100644
--- a/auto_round/mllm/template.py
+++ b/auto_round/mllm/template.py
@@ -115,6 +115,10 @@ def _register_template(
)
return TEMPLATES[model_type]
+_register_template("qwen2_vl", default_dataset="NeelNanda/pile-10k",processor=PROCESSORS["qwen2_vl"])
+_register_template("mllama", default_dataset="liuhaotian/llava", processor=PROCESSORS["hf"])
+_register_template("deepseek_vl_v2", default_dataset="NeelNanda/pile-10k",processor=PROCESSORS["deepseek_v2"])
+_register_template("mistral3", default_dataset="NeelNanda/pile-10k",processor=PROCESSORS["hf"])
def load_template(path: str):
"""Load template information from a json file."""
diff --git a/auto_round/mllm/templates/default.json b/auto_round/mllm/templates/default.json
index 321a0c485..a64017c73 100644
--- a/auto_round/mllm/templates/default.json
+++ b/auto_round/mllm/templates/default.json
@@ -10,5 +10,5 @@
"replace_tokens": null,
"extra_encode" : false,
"default_dataset": "NeelNanda/pile-10k",
- "processor": "basic"
+ "processor": "hf"
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/mllama.json b/auto_round/mllm/templates/mllama.json
deleted file mode 100644
index 2434dc4ba..000000000
--- a/auto_round/mllm/templates/mllama.json
+++ /dev/null
@@ -1,10 +0,0 @@
-{
- "model_type": "mllama",
- "format_user": "<|start_header_id|>user<|end_header_id|>\n{{content}}<|eot_id|>",
- "format_assistant": "<|start_header_id|>assistant<|end_header_id|>\n{{content}}<|eot_id|>",
- "format_system": "<|begin_of_text|><|start_header_id|>system|end_header_id|>\n{{content}}<<eot_id>",
- "default_system": "You are a helpful assistant.",
- "replace_tokens": ["<image>", "<|image|>"],
- "extra_encode" : true,
- "default_dataset": "liuhaotian/llava_conv_58k"
-}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/phi3_v.json b/auto_round/mllm/templates/phi3_v.json
index 65e7445dc..212e00d16 100644
--- a/auto_round/mllm/templates/phi3_v.json
+++ b/auto_round/mllm/templates/phi3_v.json
@@ -1,7 +1,7 @@
{
"model_type": "phi3_v",
"replace_tokens": ["<image>", "<|image_1|>"],
- "processor": "basic",
+ "processor": "hf",
"extra_encode" : false,
"default_dataset": "NeelNanda/pile-10k"
}
\ No newline at end of file
diff --git a/auto_round/mllm/templates/qwen2_vl.json b/auto_round/mllm/templates/qwen2_vl.json
deleted file mode 100644
index 6b72a0637..000000000
--- a/auto_round/mllm/templates/qwen2_vl.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
- "model_type": "qwen2_vl",
- "format_user": "<|im_start|>user\n{{content}}<|im_end|>\n",
- "format_assistant": "<|im_start|>assistant\n{{content}}<|im_end|>\n",
- "format_system": "<|im_start|>system\n{{content}}<|im_end|>\n",
- "format_observation": "<|im_start|>tool\n{{content}}<|im_end|>\n<|im_start|>assistant\n",
- "format_separator": "\n",
- "default_system": "You are a helpful assistant.",
- "replace_tokens": ["<image>", "<|vision_start|><|image_pad|><|vision_end|>"],
- "processor": "qwen2_vl",
- "extra_encode" : true,
- "default_dataset": "NeelNanda/pile-10k"
-}
\ No newline at end of file
diff --git a/auto_round/script/llm.py b/auto_round/script/llm.py
index 6b221845a..e0a5411ac 100644
--- a/auto_round/script/llm.py
+++ b/auto_round/script/llm.py
@@ -369,66 +369,20 @@ def tune(args):
if device_str is not None and "hpu" in device_str:
torch_dtype = torch.bfloat16
- is_glm = bool(re.search("chatglm", model_name.lower()))
- low_cpu_mem_usage = False
-
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
-
- model_cls = AutoModel if is_glm else AutoModelForCausalLM
-
- if args.low_cpu_mem_tmp_dir is None:
- args.low_cpu_mem_tmp_dir = os.path.join(args.output_dir, "low_cpu_mem_tmp")
- if args.low_cpu_mem_mode == 2:
- from auto_round.low_cpu_mem.utils import load_model_with_hooks
- model = load_model_with_hooks(
- model_name,
- model_cls,
- device=device_str,
- clean_weight=True,
- saved_path=args.low_cpu_mem_tmp_dir,
- torch_dtype=torch_dtype,
- trust_remote_code=not args.disable_trust_remote_code)
- elif args.low_cpu_mem_mode == 1:
- from auto_round.low_cpu_mem.utils import load_empty_model
- low_cpu_mem_usage = True
- model = load_empty_model(
- model_name,
- model_cls,
- device=device_str,
- saved_path=args.low_cpu_mem_tmp_dir,
- torch_dtype=torch_dtype,
- trust_remote_code=not args.disable_trust_remote_code)
- else:
- from auto_round.utils import _use_hpu_compile_mode
- if _use_hpu_compile_mode():
- model = model_cls.from_pretrained(
- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
- attn_implementation="eager",
- trust_remote_code=not args.disable_trust_remote_code, device_map="auto" if use_auto_mapping else None
- )
- else:
- model = model_cls.from_pretrained(
- model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
- trust_remote_code=not args.disable_trust_remote_code, device_map="auto" if use_auto_mapping else None
- )
+ from auto_round.utils import llm_load_model
+ model, tokenizer, low_cpu_mem_usage = llm_load_model(
+ model_name,
+ torch_dtype=torch_dtype,
+ use_auto_mapping=use_auto_mapping,
+ trust_remote_code=not args.disable_trust_remote_code,
+ device=device_str,
+ low_cpu_mem_mode=args.low_cpu_mem_mode,
+ low_cpu_mem_tmp_dir=args.low_cpu_mem_tmp_dir)
from auto_round import AutoRound, AutoRoundAdam
- model = model.eval()
seqlen = args.seqlen
- if args.model_dtype != None:
- try:
- if args.model_dtype == "float16" or args.model_dtype == "fp16":
- model = model.to(torch.float16)
- elif args.model_dtype == "bfloat16" or args.model_dtype == "bfp16" or args.model_dtype == "bf16":
- model = model.to(torch.bfloat16)
- elif args.model_dtype == "float32" or args.model_dtype == "fp32":
- model = model.to(torch.float32)
- except:
- logger.error("please use more device to fit the device or just use one device")
- exit()
-
if hasattr(tokenizer, "model_max_length"):
if tokenizer.model_max_length < seqlen:
logger.info(
diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.py
index 4c74a1732..6d06aca1c 100644
--- a/auto_round/script/mllm.py
+++ b/auto_round/script/mllm.py
@@ -322,70 +322,17 @@ def tune(args):
torch_dtype = torch.bfloat16
# load_model
- processor, image_processor = None, None
- if "deepseek-vl2" in model_name.lower():
- from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM # pylint: disable=E0401
- processor = DeepseekVLV2Processor.from_pretrained(model_name)
- tokenizer = processor.tokenizer
- model: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(
- model_name,
- trust_remote_code=not args.disable_trust_remote_code,
- torch_dtype=torch_dtype,
- device_map="auto" if use_auto_mapping else None)
- model_type = "deepseek_vl_v2"
- else:
- config = AutoConfig.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
- if "llava" in model_name and config.architectures[0] != "LlavaForConditionalGeneration":
- from llava.model.builder import load_pretrained_model # pylint: disable=E0401
- tokenizer, model, image_processor, _ = load_pretrained_model(
- model_name, model_base=None, model_name=model_name, torch_dtype=torch_dtype)
- model_type = "llava"
- else:
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
- processor = AutoProcessor.from_pretrained(model_name, trust_remote_code=not args.disable_trust_remote_code)
- model_type = config.model_type
- if "llava" in model_type:
- from transformers import LlavaForConditionalGeneration
- cls = LlavaForConditionalGeneration
- elif "qwen2_vl" in model_type:
- from transformers import Qwen2VLForConditionalGeneration
- cls = Qwen2VLForConditionalGeneration
- elif "mllama" in model_type:
- from transformers import MllamaForConditionalGeneration
- cls = MllamaForConditionalGeneration
- elif "gemma3" in model_type:
- from transformers import Gemma3ForConditionalGeneration
- cls = Gemma3ForConditionalGeneration
- elif "mistral3" in model_type:
- from transformers import Mistral3ForConditionalGeneration
- cls = Mistral3ForConditionalGeneration
- else:
- cls = AutoModelForCausalLM
-
- model = cls.from_pretrained(
- model_name,
- trust_remote_code=not args.disable_trust_remote_code,
- torch_dtype=torch_dtype,
- device_map="auto" if use_auto_mapping else None)
- if "cogvlm2" in model_name:
- model.config.model_type = "cogvlm2"
+ from auto_round.utils import mllm_load_model
+ model, processor, tokenizer, image_processor = mllm_load_model(
+ model_name,
+ torch_dtype=torch_dtype,
+ use_auto_mapping=use_auto_mapping,
+ trust_remote_code=not args.disable_trust_remote_code)
from auto_round import AutoRoundMLLM
model = model.eval()
- if args.model_dtype != None:
- try:
- if args.model_dtype == "float16" or args.model_dtype == "fp16":
- model = model.to(torch.float16)
- elif args.model_dtype == "bfloat16" or args.model_dtype == "bfp16" or args.model_dtype == "bf16":
- model = model.to(torch.bfloat16)
- elif args.model_dtype == "float32" or args.model_dtype == "fp32":
- model = model.to(torch.float32)
- except:
- logger.error("please use more device to fit the device or just use one device")
- exit()
-
round = AutoRoundMLLM
layer_config = {}
@@ -399,7 +346,6 @@ def tune(args):
##TODO gptq could support some mixed precision config
logger.warning(f"mixed precision exporting does not support {format} currently")
- layer_config = {}
if args.fp_layers != "":
fp_layers = args.fp_layers.replace(" ", "").split(",")
for n, m in model.named_modules():
@@ -461,6 +407,11 @@ def tune(args):
enable_torch_compile = True if "--enable_torch_compile" in sys.argv else False
+ model_kwargs = {
+ "use_auto_mapping": use_auto_mapping,
+ "trust_remote_code": not args.disable_trust_remote_code,
+ "model_dtype": args.model_dtype
+ }
autoround = round(
model,
tokenizer,
@@ -494,7 +445,9 @@ def tune(args):
not_use_best_mse=args.not_use_best_mse,
to_quant_block_names=args.to_quant_block_names,
enable_torch_compile=enable_torch_compile,
- device_map=args.device_map)
+ device_map=args.device_map,
+ model_kwargs=model_kwargs
+ )
model, _ = autoround.quantize()
model.eval()
@@ -510,8 +463,9 @@ def tune(args):
for format_ in format_list:
eval_folder = f'{export_dir}-{format_}'
safe_serialization = True
- if "phi3_v" in model_type:
- safe_serialization = False
+ if hasattr(autoround.model, "config") and hasattr(autoround.model.config, "model_type"):
+ if "phi3_v" in autoround.model.config.model_type:
+ safe_serialization = False
autoround.save_quantized(eval_folder, format=format_, inplace=inplace, safe_serialization=safe_serialization)
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 03aa019a7..de8eb4ea7 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -28,6 +28,14 @@ SUPPORT_ONLY_TEXT_MODELS = [
"idefics3"
]
+def _handle_special_model(model):
+ if model.config.model_type == "deepseek_vl_v2":
+ from auto_round.special_model_handler import _deepseek_vl2_forward
+ from functools import partial
+ model.forward = partial(_deepseek_vl2_forward, model)
+ return model
+
+
def _get_deepseek_vl2_multimodal_block(model, quant_vision=False):
model.forward = model.language.forward
block_names = []
@@ -41,6 +49,46 @@ SPECIAL_MULTIMODAL_BLOCK = {
"deepseek_vl_v2": _get_deepseek_vl2_multimodal_block
}
+def _deepseek_vl2_forward(
+ model,
+ input_ids = None,
+
+ position_ids = None,
+ attention_mask = None,
+ past_key_values = None,
+ inputs_embeds = None,
+
+ images = None,
+ images_seq_mask = None,
+ images_spatial_crop = None,
+
+ labels = None,
+ use_cache = None,
+ output_attentions = None,
+ output_hidden_states = None,
+ return_dict = None,
+ cache_position = None,
+ **kwargs
+ ):
+ inputs_embeds = model.prepare_inputs_embeds(
+ input_ids=input_ids,
+ images=images,
+ images_seq_mask=images_seq_mask,
+ images_spatial_crop=images_spatial_crop,
+ )
+ return model.language(
+ input_ids=None,
+ attention_mask=attention_mask,
+ position_ids=position_ids,
+ past_key_values=past_key_values,
+ inputs_embeds=inputs_embeds,
+ labels=labels,
+ use_cache=use_cache,
+ output_attentions=output_attentions,
+ output_hidden_states=output_hidden_states,
+ return_dict=return_dict,
+ cache_position=cache_position)
+
def to_device(input, device=torch.device("cpu")):
"""Moves input data to the specified device.
@@ -103,7 +151,7 @@ def reset_params(inputs):
"""
if "use_cache" in inputs.keys(): # Not storing kv cache
inputs['use_cache'] = False
-
+
def check_skippable_keywords(key):
"""
@@ -113,7 +161,7 @@ def check_skippable_keywords(key):
if cache_key not in key:
return True
return False
-
+
def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):
"""
@@ -126,4 +174,3 @@ def check_mllm_model_batch(model, batch_size, gradient_accumulate_steps=1):
f"batch_size=1. As an alternative, set the gradient_accumulate_steps={accumulate_steps}")
return 1, accumulate_steps
return batch_size, gradient_accumulate_steps
-
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 9e1b64511..ea9779893 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -421,7 +421,7 @@ def get_block_names(model, quant_vision=False):
for n, m in target_m[1].named_children():
block_names[i].append(target_m[0] + "." + n)
return block_names
-
+
if quant_vision or not is_pure_text_model(model):
return _get_vlm_block_names(model, quant_vision=quant_vision)
else:
@@ -1229,7 +1229,7 @@ def _gguf_args_check(args):
sys.exit(-1)
else:
pre_dq_format = format
-
+
if os.path.isdir(args.model):
from pathlib import Path
from auto_round.export.export_to_gguf.convert import Model
@@ -1265,6 +1265,144 @@ def _gguf_args_check(args):
return args
+
+def _to_model_dtype(model, model_dtype):
+ if model_dtype != None:
+ try:
+ if model_dtype == "float16" or model_dtype == "fp16":
+ model = model.to(torch.float16)
+ elif model_dtype == "bfloat16" or model_dtype == "bfp16" or model_dtype == "bf16":
+ model = model.to(torch.bfloat16)
+ elif model_dtype == "float32" or model_dtype == "fp32":
+ model = model.to(torch.float32)
+ except:
+ logger.error("please use more device to fit the device or just use one device")
+ exit()
+ return model
+
+def llm_load_model(
+ pretrained_model_name_or_path,
+ torch_dtype="auto",
+ use_auto_mapping=True,
+ trust_remote_code=True,
+ model_dtype=None,
+ device="cpu",
+ low_cpu_mem_mode=0,
+ low_cpu_mem_tmp_dir = None,
+ **kwargs):
+ from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM
+
+ is_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))
+ low_cpu_mem_usage = False
+
+ tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
+
+ model_cls = AutoModel if is_glm else AutoModelForCausalLM
+
+ if low_cpu_mem_tmp_dir is None:
+ low_cpu_mem_tmp_dir = "low_cpu_mem_tmp"
+ if low_cpu_mem_mode == 2:
+ from auto_round.low_cpu_mem.utils import load_model_with_hooks
+ model = load_model_with_hooks(
+ pretrained_model_name_or_path,
+ model_cls,
+ device=device,
+ clean_weight=True,
+ saved_path=low_cpu_mem_tmp_dir,
+ torch_dtype=torch_dtype,
+ trust_remote_code=trust_remote_code)
+ elif low_cpu_mem_mode == 1:
+ from auto_round.low_cpu_mem.utils import load_empty_model
+ low_cpu_mem_usage = True
+ model = load_empty_model(
+ pretrained_model_name_or_path,
+ model_cls,
+ device=device,
+ saved_path=low_cpu_mem_tmp_dir,
+ torch_dtype=torch_dtype,
+ trust_remote_code=trust_remote_code)
+ else:
+ from auto_round.utils import _use_hpu_compile_mode
+ if _use_hpu_compile_mode():
+ model = model_cls.from_pretrained(
+ pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
+ attn_implementation="eager",
+ trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None
+ )
+ else:
+ model = model_cls.from_pretrained(
+ pretrained_model_name_or_path, low_cpu_mem_usage=True, torch_dtype=torch_dtype,
+ trust_remote_code=trust_remote_code, device_map="auto" if use_auto_mapping else None
+ )
+
+ model = model.eval()
+ model = _to_model_dtype(model, model_dtype)
+
+ return model, tokenizer, low_cpu_mem_usage
+
+
+def mllm_load_model(
+ pretrained_model_name_or_path,
+ torch_dtype="auto",
+ use_auto_mapping=True,
+ trust_remote_code=True,
+ model_dtype=None,
+ **kwargs):
+ import json
+ import transformers
+ from transformers import AutoProcessor, AutoTokenizer, AutoModelForCausalLM
+ from huggingface_hub import HfApi, HfFileSystem
+
+ if os.path.isdir(pretrained_model_name_or_path):
+ config = json.load(open(os.path.join(pretrained_model_name_or_path, "config.json")))
+ else:
+ hf_file = HfFileSystem()
+ config = json.load(hf_file.open(pretrained_model_name_or_path + "/config.json"))
+
+ if "model_type" in config:
+ model_type = config["model_type"]
+ else:
+ model_type = None
+
+ processor, image_processor = None, None
+ if "deepseek_vl_v2" == model_type:
+ from deepseek_vl2.models import DeepseekVLV2Processor, DeepseekVLV2ForCausalLM # pylint: disable=E0401
+ processor = DeepseekVLV2Processor.from_pretrained(pretrained_model_name_or_path)
+ tokenizer = processor.tokenizer
+ model: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(
+ pretrained_model_name_or_path,
+ trust_remote_code=trust_remote_code,
+ torch_dtype=torch_dtype,
+ device_map="auto" if use_auto_mapping else None)
+ else:
+ architectures = config["architectures"][0]
+ if architectures == "LlavaLlamaForCausalLM":
+ from llava.model.builder import load_pretrained_model # pylint: disable=E0401
+ tokenizer, model, image_processor, _ = load_pretrained_model(
+ pretrained_model_name_or_path,
+ model_base=None,
+ model_name=pretrained_model_name_or_path,
+ torch_dtype=torch_dtype)
+ else:
+ if hasattr(transformers, architectures):
+ cls = getattr(transformers, architectures)
+ else:
+ cls = AutoModelForCausalLM
+ model = cls.from_pretrained(
+ pretrained_model_name_or_path,
+ trust_remote_code=trust_remote_code,
+ torch_dtype=torch_dtype,
+ device_map="auto" if use_auto_mapping else None)
+ tokenizer = AutoTokenizer.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
+ processor = AutoProcessor.from_pretrained(
+ pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
+
+ model = model.eval()
+ model = _to_model_dtype(model, model_dtype)
+
+ return model, processor, tokenizer, image_processor
+
def is_pure_text_model(model):
"""verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl, """
if hasattr(model,"config") and hasattr(model.config, "vision_config"):
@@ -1280,4 +1418,4 @@ def is_pure_text_model(model):
return False
if "img" in str(module.__class__).lower():
return False
- return True
+ return True
diff --git a/test/test_block_names.py b/test/test_block_names.py
index c98885e50..b828a1a3a 100644
--- a/test/test_block_names.py
+++ b/test/test_block_names.py
@@ -174,6 +174,19 @@ class TestQuantizationBlocks(unittest.TestCase):
assert quant_config.to_quant_block_names is not None
shutil.rmtree("./saved", ignore_errors=True)
+
+ def test_mm_block_name(self):
+ from auto_round.utils import get_block_names
+ from transformers import Qwen2VLForConditionalGeneration
+ model = Qwen2VLForConditionalGeneration.from_pretrained(
+ self.model_name, trust_remote_code=True, device_map="auto")
+ block_name = get_block_names(model, quant_vision=True)
+ self.assertTrue(len(block_name) == 2)
+ self.assertTrue(all(["visual.merger.mlp" not in n for n in block_name]))
+ block_name = get_block_names(model, quant_vision=False)
+ self.assertTrue(len(block_name) == 1)
+ self.assertTrue(block_name == get_block_names(model))
+
def test_moe(self):
from auto_round.utils import get_block_names
model_name = "Qwen/Qwen1.5-MoE-A2.7B"
@@ -205,6 +218,7 @@ class TestQuantizationBlocks(unittest.TestCase):
# quant_config = model.config.quantization_config
# assert quant_config.to_quant_block_names is not None
+
if __name__ == "__main__":
diff --git a/test/test_mllm.py b/test/test_mllm.py
index 68b27752e..da811622c 100644
--- a/test/test_mllm.py
+++ b/test/test_mllm.py
@@ -15,9 +15,13 @@ class FakeDataLoader:
self.batch_size = 1
self.data = {
- "text": [{'role': 'user',
- 'content': '<|vision_start|><|image_pad|><|vision_end|>\nWhat are the colors of the bus in the image?'},
- {'role': 'assistant', 'content': 'The bus in the image is white and red.'}],
+ "text": [{
+ 'role': 'user',
+ 'content': '<image>\nWhat are the colors of the bus in the image?'
+ }, {
+ 'role': 'assistant',
+ 'content': 'The bus in the image is white and red.'
+ }],
"image": "http://images.cocodataset.org/train2017/000000033471.jpg"
}
@@ -46,10 +50,10 @@ class TestAutoRoundMLLM(unittest.TestCase):
self.model_name, trust_remote_code=True, device_map="auto")
bits, group_size = 4, 128
autoround = AutoRoundMLLM(
- model, tokenizer, processor=processor,
+ model, tokenizer, processor=processor,
bits=bits, group_size=group_size,
nsamples=1,
- batch_size=1, iters=2, dataset=self.dataset,seqlen=256)
+ batch_size=1, iters=2, dataset=self.dataset,seqlen=10)
autoround.quantize()
autoround.save_quantized("./saved/", format="auto_gptq", inplace=False)
autoround.save_quantized("./saved/", format="auto_round", inplace=False)
@@ -64,10 +68,10 @@ class TestAutoRoundMLLM(unittest.TestCase):
model, tokenizer, processor=processor,
bits=bits, group_size=group_size,
nsamples=5,
- batch_size=3, iters=2, dataset=self.dataset, quant_nontext_module=True, seqlen=256)
+ batch_size=3, iters=2, dataset=self.dataset, quant_nontext_module=True, seqlen=10)
autoround.quantize()
autoround.save_quantized("./saved/", format="auto_round", inplace=True)
-
+
def test_quant_block_names(self):
from auto_round.utils import get_block_names,find_matching_blocks
tokenizer = AutoTokenizer.from_pretrained(self.model_name)
@@ -79,7 +83,7 @@ class TestAutoRoundMLLM(unittest.TestCase):
all_blocks = get_block_names(model, quant_vision=True)
blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)
assert target_blocks == blocks
-
+
def test_dataset_check(self):
from auto_round.mllm.mllm_dataset import MLLM_DATASET
class Myclass:
@@ -88,7 +92,7 @@ class TestAutoRoundMLLM(unittest.TestCase):
self.assertEqual(len(dataset.questions), 32)
dataset = MLLM_DATASET['liuhaotian/llava'](template=Myclass(), model=None, tokenzier=None, dataset_path="liuhaotian/llava", seqlen=2048, nsamples=512)
self.assertEqual(len(dataset.questions), 512)
-
+
def test_diff_dataset(self):
tokenizer = AutoTokenizer.from_pretrained(self.model_name)
processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)
@@ -113,8 +117,6 @@ class TestAutoRoundMLLM(unittest.TestCase):
self.assertTrue(is_pure_text_model(model))
+
if __name__ == "__main__":
unittest.main()
-
-
-
diff --git a/test_cuda/test_vlms.py b/test_cuda/test_vlms.py
index ab0438e0d..c32a061a8 100644
--- a/test_cuda/test_vlms.py
+++ b/test_cuda/test_vlms.py
@@ -193,6 +193,34 @@ class TestAutoRound(unittest.TestCase):
f"--quant_nontext_module --iters 1 --nsamples 1 --output_dir {absolute_path}")
self.phi3_infernece(absolute_path)
shutil.rmtree(absolute_path, ignore_errors=True)
+
+ def test_mm_block_name(self):
+ from auto_round.utils import get_block_names
+
+ model_name = "/models/deepseek-vl2-tiny"
+ from deepseek_vl2.models import DeepseekVLV2ForCausalLM
+ model = DeepseekVLV2ForCausalLM.from_pretrained(
+ model_name, trust_remote_code=True, device_map="auto")
+ block_name = get_block_names(model, quant_vision=True)
+ self.assertTrue(len(block_name) == 3)
+ self.assertTrue(any(["projector.layers.1" not in n for n in block_name]))
+ block_name = get_block_names(model, quant_vision=False)
+ self.assertTrue(len(block_name) == 1)
+ self.assertTrue(get_block_names(model) == block_name)
+
+ model_name = "/models/Llama-3.2-11B-Vision-Instruct/"
+ from transformers import MllamaForConditionalGeneration
+ model = MllamaForConditionalGeneration.from_pretrained(
+ model_name, trust_remote_code=True, device_map="auto")
+ block_name = get_block_names(model, quant_vision=True)
+ self.assertTrue(len(block_name) == 3)
+ self.assertTrue(any(["vision_model.global_transformer.layers.0" not in n for n in block_name]))
+ self.assertTrue(any(["vision_model.transformer.layers.0" not in n for n in block_name]))
+ block_name = get_block_names(model, quant_vision=False)
+ self.assertTrue(len(block_name) == 1)
+ self.assertTrue(get_block_names(model) == block_name)
+
+