Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

code optimization of vlm (#704)

Heng Guoef4af0f04a07
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 8f93378d4..bd3237d7a 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1415,6 +1415,7 @@ class AutoRound(object):
f"Expected exactly one packing format when 'is_packing_immediate' is True, "
f"but got {len(self.formats)} formats."
)
+ pbar.close()
self.quant_layers(layer_names, all_inputs) ##TODO pack layer immediately
@@ -2515,7 +2516,6 @@ class AutoRound(object):
PACKING_LAYER_WITH_FORMAT[target_backend](tmp_m.tmp_name, self.model, self.formats[0])
pbar.set_description("Quantizing done")
pbar.update(1)
- pbar.close()
self.model = mv_module_from_gpu(self.model, self.low_cpu_mem_usage)
for n, m in self.model.named_modules():
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 8cba7531a..4aaa562f5 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -165,6 +165,7 @@ class AutoRoundMLLM(AutoRound):
model_kwargs: dict = None,
**kwargs,
):
+ quant_nontext_module = self._check_quant_nontext(layer_config, quant_nontext_module)
all_blocks = get_block_names(model, quant_nontext_module)
self.quant_block_list = find_matching_blocks(model, all_blocks, to_quant_block_names)
if to_quant_block_names is None:
@@ -196,21 +197,18 @@ class AutoRoundMLLM(AutoRound):
from ..calib_dataset import CALIB_DATASETS
from .mllm_dataset import MLLM_DATASET
- if isinstance(dataset, str):
- if quant_nontext_module or (
- dataset in CALIB_DATASETS.keys()
- and not _only_text_test(model, tokenizer, device, self.template.model_type)
- ):
- if quant_nontext_module:
- logger.warning(
- "Text only dataset cannot be used for calibrating non-text modules,"
- "switching to liuhaotian/llava_conv_58k"
- )
- else:
- logger.warning(
- f"{model.config.model_type} not support for {dataset},"
- " will use liuhaotian/llava_conv_58k with default config as an alternative."
- )
+ if isinstance(dataset, str) and dataset in CALIB_DATASETS.keys():
+ if quant_nontext_module:
+ logger.warning(
+ "Text only dataset cannot be used for calibrating non-text modules,"
+ " switching to liuhaotian/llava_conv_58k"
+ )
+ dataset = "liuhaotian/llava_conv_58k"
+ elif not _only_text_test(model, tokenizer, device, self.template.model_type):
+ logger.warning(
+ f"{model.config.model_type} does not support for {dataset},"
+ " will use liuhaotian/llava_conv_58k with default config as an alternative."
+ )
dataset = "liuhaotian/llava_conv_58k"
if dataset in MLLM_DATASET.keys():
@@ -442,3 +440,14 @@ class AutoRoundMLLM(AutoRound):
output_dir=output_dir, format=format, inplace=inplace, processor=self.processor, **kwargs
)
return compressed_model
+
+ def _check_quant_nontext(self, layer_config, quant_nontext_module):
+ if not layer_config:
+ return quant_nontext_module
+ from auto_round.mllm.utils import VISUAL_KEYS
+
+ for layer_name in layer_config.keys():
+ for vlm_key in VISUAL_KEYS:
+ if vlm_key in layer_name:
+ return True
+ return quant_nontext_module
diff --git a/auto_round/mllm/processor.py b/auto_round/mllm/processor.py
index bf48f5556..ac021e13b 100644
--- a/auto_round/mllm/processor.py
+++ b/auto_round/mllm/processor.py
@@ -89,6 +89,19 @@ class HFProcessor(BasicProcessor):
def __init__(self):
self.process_func = self._process_v1
+ def post_init(self, model, tokenizer, processor=None, image_processor=None, use_rtn=False, **kwargs):
+ assert tokenizer is not None, "tokenizer should not be None"
+ assert processor is not None, "processor should not be None"
+ self.model = model
+ self.tokenizer = tokenizer
+ self.processor = processor
+ if image_processor is not None:
+ self.image_processor = image_processor
+ else:
+ self.image_processor = self.default_image_processor
+ self.use_rtn = use_rtn
+ self.check_image_processor()
+
def _process_v1(self, messages, image):
"""support models: Qwen2-VL, gemma-3, granite-vision-3.2, Aria"""
conversation = []
diff --git a/auto_round/mllm/utils.py b/auto_round/mllm/utils.py
index 029080d68..f9a012627 100644
--- a/auto_round/mllm/utils.py
+++ b/auto_round/mllm/utils.py
@@ -21,6 +21,19 @@ from ..utils import LazyImport
PIL = LazyImport("PIL")
from PIL import Image # pylint: disable=E0401
+VISUAL_KEYS = [
+ "thinker",
+ "visual",
+ "audio",
+ "talker",
+ "token2wav",
+ "multi_modal_projector",
+ "vision_tower",
+ "multimodal_projector",
+ "vision_model",
+ "model.connector",
+]
+
def _extract_data_dir(dir_path: str):
if os.path.isdir(dir_path):