PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix seqlen bug and calib slow of mllm tuning (#871)
Heng Guo7210791955b8
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex f56109525..97f7e8682 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -2226,6 +2226,7 @@ class BaseCompressor(object):and len(layer_names) == 0and not self.has_qlayer_outside_blockand (last_cache_name is None or last_cache_name in block_names)+ and getattr(self, "mllm", False) is False):# low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPUall_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.pyindex 63199b081..6fcb50c2c 100644--- a/auto_round/compressors/mllm/compressor.py+++ b/auto_round/compressors/mllm/compressor.py@@ -12,10 +12,13 @@# See the License for the specific language governing permissions and# limitations under the License.+import tracebackfrom copy import deepcopyfrom typing import Union+import accelerateimport torch+from accelerate.big_modeling import dispatch_model, infer_auto_device_mapfrom tqdm import tqdmfrom auto_round.compressors.base import BaseCompressor@@ -30,13 +33,16 @@ from auto_round.special_model_handler import (_handle_special_model,)from auto_round.utils import (+ _is_fp8_model,check_to_quantized,clear_memory,detect_device,extract_block_names_to_str,find_matching_blocks,get_block_names,+ get_max_vram,mllm_load_model,+ mv_module_from_gpu,to_device,to_dtype,)@@ -284,7 +290,12 @@ class MLLMCompressor(BaseCompressor):"""if isinstance(self.dataset, str):dataset = self.dataset.replace(" ", "")- self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_mllm_dataloader(+ (+ self.dataloader,+ self.batch_size,+ self.seqlen,+ self.gradient_accumulate_steps,+ ) = get_mllm_dataloader(template=self.template,model=self.model,tokenizer=self.tokenizer,diff --git a/auto_round/compressors/mllm/dataset.py b/auto_round/compressors/mllm/dataset.pyindex 26f6ef142..349b8a33f 100644--- a/auto_round/compressors/mllm/dataset.py+++ b/auto_round/compressors/mllm/dataset.py@@ -62,6 +62,7 @@ class LlavaDataset(Dataset):}_COCO_DATA_URL = "http://images.cocodataset.org/train2017/"IMAGE_TOKEN = "<image>"+ MAX_SUPPORT_SEQLEN = 512def __init__(self,@@ -233,7 +234,13 @@ def get_mllm_dataloader()if os.path.isfile(dataset) or dataset in MLLM_DATASET.keys():- dataset = MLLM_DATASET["liuhaotian/llava"](+ if seqlen > MLLM_DATASET[dataset].MAX_SUPPORT_SEQLEN:+ logger.warning(+ f"seqlen({seqlen}) is greater than the maximum length supported by the {dataset},"+ f" reset to {MLLM_DATASET[dataset].MAX_SUPPORT_SEQLEN}"+ )+ seqlen = 512+ dataset = MLLM_DATASET[dataset](template, model, tokenizer, dataset, extra_data_dir, seqlen=seqlen, truncation=truncation, nsamples=nsamples)@@ -244,7 +251,7 @@ def get_mllm_dataloader(set_seed(seed)dataloader_params = {"batch_size": bs, "shuffle": True, "collate_fn": dataset.template.processor.data_collator}- return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_steps+ return DataLoader(dataset, **dataloader_params), bs, seqlen, gradient_accumulate_stepselse:# try to load text calibration datasetfrom auto_round.calib_dataset import get_dataloader@@ -256,4 +263,4 @@ def get_mllm_dataloader(" switching to liuhaotian/llava_conv_58k")exit(-1)- return dataloader, bs, gradient_accumulate_steps+ return dataloader, bs, seqlen, gradient_accumulate_steps