PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
code optimization of vlm (#704)
Heng Guoef4af0f04a07
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 8f93378d4..bd3237d7a 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1415,6 +1415,7 @@ class AutoRound(object):f"Expected exactly one packing format when 'is_packing_immediate' is True, "f"but got {len(self.formats)} formats.")+ pbar.close()self.quant_layers(layer_names, all_inputs) ##TODO pack layer immediately@@ -2515,7 +2516,6 @@ class AutoRound(object):PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])pbar.set_description("Quantizing done")pbar.update(1)- pbar.close()self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)for n, m in self.model.named_modules():diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 8cba7531a..4aaa562f5 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -165,6 +165,7 @@ class AutoRoundMLLM(AutoRound):model_kwargs: dict = None,**kwargs,):+ quant_nontext_module = self._check_quant_nontext(layer_config, quant_nontext_module)all_blocks = get_block_names(model, quant_nontext_module)self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)if to_quant_block_names is None:@@ -196,21 +197,18 @@ class AutoRoundMLLM(AutoRound):from ..calib_dataset import CALIB_DATASETSfrom .mllm_dataset import MLLM_DATASET- if isinstance(dataset, str):- if quant_nontext_module or (- dataset in CALIB_DATASETS.keys()- and not _only_text_test(model, tokenizer, device, self.template.model_type)- ):- if quant_nontext_module:- logger.warning(- "Text only dataset cannot be used for calibrating non-text modules,"- "switching to liuhaotian/llava_conv_58k"- )- else:- logger.warning(- f"{model.config.model_type} not support for {dataset},"- " will use liuhaotian/llava_conv_58k with default config as an alternative."- )+ if isinstance(dataset, str) and dataset in CALIB_DATASETS.keys():+ if quant_nontext_module:+ logger.warning(+ "Text only dataset cannot be used for calibrating non-text modules,"+ " switching to liuhaotian/llava_conv_58k"+ )+ dataset = "liuhaotian/llava_conv_58k"+ elif not _only_text_test(model, tokenizer, device, self.template.model_type):+ logger.warning(+ f"{model.config.model_type} does not support for {dataset},"+ " will use liuhaotian/llava_conv_58k with default config as an alternative."+ )dataset = "liuhaotian/llava_conv_58k"if dataset in MLLM_DATASET.keys():@@ -442,3 +440,14 @@ class AutoRoundMLLM(AutoRound):output_dir=output_dir, format=format, inplace=inplace, processor=self.processor, **kwargs)return compressed_model++ def _check_quant_nontext(self, layer_config, quant_nontext_module):+ if not layer_config:+ return quant_nontext_module+ from auto_round.mllm.utils import VISUAL_KEYS++ for layer_name in layer_config.keys():+ for vlm_key in VISUAL_KEYS:+ if vlm_key in layer_name:+ return True+ return quant_nontext_modulediff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.pyindex bf48f5556..ac021e13b 100644--- a/auto_round/mllm/processor.py+++ b/auto_round/mllm/processor.py@@ -89,6 +89,19 @@ class HFProcessor(BasicProcessor):def __init__(self):self.process_func = self._process_v1+ def post_init(self, model, tokenizer, processor=None, image_processor=None, use_rtn=False, **kwargs):+ assert tokenizer is not None, "tokenizer should not be None"+ assert processor is not None, "processor should not be None"+ self.model = model+ self.tokenizer = tokenizer+ self.processor = processor+ if image_processor is not None:+ self.image_processor = image_processor+ else:+ self.image_processor = self.default_image_processor+ self.use_rtn = use_rtn+ self.check_image_processor()+def _process_v1(self, messages, image):"""support models: Qwen2-VL, gemma-3, granite-vision-3.2, Aria"""conversation = []diff --git a/auto_round/mllm/utils.py b/auto_round/mllm/utils.pyindex 029080d68..f9a012627 100644--- a/auto_round/mllm/utils.py+++ b/auto_round/mllm/utils.py@@ -21,6 +21,19 @@ from ..utils import LazyImportPIL = LazyImport("PIL")from PIL import Image # pylint: disable=E0401+VISUAL_KEYS = [+ "thinker",+ "visual",+ "audio",+ "talker",+ "token2wav",+ "multi_modal_projector",+ "vision_tower",+ "multimodal_projector",+ "vision_model",+ "model.connector",+]+def _extract_data_dir(dir_path: str):if os.path.isdir(dir_path):