Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix the bug of test model support for test-only (#328)

Heng Guodebcf830e1d4
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.py
index 3d95f26b5..485aa0d15 100644
--- a/auto_round/mllm/autoround_mllm.py
+++ b/auto_round/mllm/autoround_mllm.py
@@ -31,6 +31,18 @@ from ..low_cpu_mem.utils import get_layers_before_block
from ..calib_dataset import CALIB_DATASETS
+def _only_text_test(model, tokenizer):
+ """Test if the model whether can use text-only datasets."""
+ try:
+ text = ["only text", "test"]
+ tokenizer.padding_side = 'left'
+ inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True).to(model.device)
+ model(**inputs)
+ return True
+ except:
+ return False
+
+
class AutoRoundMLLM(AutoRound):
"""Class for automatic rounding-based quantization with MLLMs.
@@ -137,7 +149,7 @@ class AutoRoundMLLM(AutoRound):
truncation = True if dataset in CALIB_DATASETS.keys() else False
self.truncation = truncation
- if dataset in CALIB_DATASETS.keys() and not self._only_text_test():
+ if dataset in CALIB_DATASETS.keys() and not _only_text_test(model, tokenizer):
logger.warning(f"{model.config.model_type} not support for {dataset},"
" will use liuhaotian/llava_conv_58k with default config as an alternative.")
dataset = "liuhaotian/llava_conv_58k"
@@ -187,15 +199,7 @@ class AutoRoundMLLM(AutoRound):
)
- def _only_text_test(self):
- """Test if the model whether can use text-only datasets."""
- try:
- text = ["only text", "test"]
- inputs = self.tokenizer(text, return_tensors="pt").to(self.model.device)
- self.model(**inputs)
- return True
- except:
- return False
+
def calib(self, nsamples, bs):