PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix the bug of test model support for test-only (#328)
Heng Guodebcf830e1d4
diff --git a/auto_round/mllm/autoround_mllm.py b/auto_round/mllm/autoround_mllm.pyindex 3d95f26b5..485aa0d15 100644--- a/auto_round/mllm/autoround_mllm.py+++ b/auto_round/mllm/autoround_mllm.py@@ -31,6 +31,18 @@ from ..low_cpu_mem.utils import get_layers_before_blockfrom ..calib_dataset import CALIB_DATASETS+def _only_text_test(model, tokenizer):+ """Test if the model whether can use text-only datasets."""+ try:+ text = ["only text", "test"]+ tokenizer.padding_side = 'left'+ inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True).to(model.device)+ model(**inputs)+ return True+ except:+ return False++class AutoRoundMLLM(AutoRound):"""Class for automatic rounding-based quantization with MLLMs.@@ -137,7 +149,7 @@ class AutoRoundMLLM(AutoRound):truncation = True if dataset in CALIB_DATASETS.keys() else Falseself.truncation = truncation- if dataset in CALIB_DATASETS.keys() and not self._only_text_test():+ if dataset in CALIB_DATASETS.keys() and not _only_text_test(model, tokenizer):logger.warning(f"{model.config.model_type} not support for {dataset},"" will use liuhaotian/llava_conv_58k with default config as an alternative.")dataset = "liuhaotian/llava_conv_58k"@@ -187,15 +199,7 @@ class AutoRoundMLLM(AutoRound):)- def _only_text_test(self):- """Test if the model whether can use text-only datasets."""- try:- text = ["only text", "test"]- inputs = self.tokenizer(text, return_tensors="pt").to(self.model.device)- self.model(**inputs)- return True- except:- return False+def calib(self, nsamples, bs):