Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix seqlen bug and calib slow of mllm tuning (#871)

Heng Guo7210791955b8
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index f56109525..97f7e8682 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -2226,6 +2226,7 @@ class BaseCompressor(object):
and len(layer_names) == 0
and not self.has_qlayer_outside_block
and (last_cache_name is None or last_cache_name in block_names)
+ and getattr(self, "mllm", False) is False
):
# low_gpu_mem_usage or calibrate only the embedding layer, which is also very fast on CPU
all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)
diff --git a/auto_round/compressors/mllm/compressor.py b/auto_round/compressors/mllm/compressor.py
index 63199b081..6fcb50c2c 100644
--- a/auto_round/compressors/mllm/compressor.py
+++ b/auto_round/compressors/mllm/compressor.py
@@ -12,10 +12,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import traceback
from copy import deepcopy
from typing import Union
+import accelerate
import torch
+from accelerate.big_modeling import dispatch_model, infer_auto_device_map
from tqdm import tqdm
from auto_round.compressors.base import BaseCompressor
@@ -30,13 +33,16 @@ from auto_round.special_model_handler import (
_handle_special_model,
)
from auto_round.utils import (
+ _is_fp8_model,
check_to_quantized,
clear_memory,
detect_device,
extract_block_names_to_str,
find_matching_blocks,
get_block_names,
+ get_max_vram,
mllm_load_model,
+ mv_module_from_gpu,
to_device,
to_dtype,
)
@@ -284,7 +290,12 @@ class MLLMCompressor(BaseCompressor):
"""
if isinstance(self.dataset, str):
dataset = self.dataset.replace(" ", "")
- self.dataloader, self.batch_size, self.gradient_accumulate_steps = get_mllm_dataloader(
+ (
+ self.dataloader,
+ self.batch_size,
+ self.seqlen,
+ self.gradient_accumulate_steps,
+ ) = get_mllm_dataloader(
template=self.template,
model=self.model,
tokenizer=self.tokenizer,
diff --git a/auto_round/compressors/mllm/dataset.py b/auto_round/compressors/mllm/dataset.py
index 26f6ef142..349b8a33f 100644
--- a/auto_round/compressors/mllm/dataset.py
+++ b/auto_round/compressors/mllm/dataset.py
@@ -62,6 +62,7 @@ class LlavaDataset(Dataset):
}
_COCO_DATA_URL = "http://images.cocodataset.org/train2017/"
IMAGE_TOKEN = "<image>"
+ MAX_SUPPORT_SEQLEN = 512
def __init__(
self,
@@ -233,7 +234,13 @@ def get_mllm_dataloader(
)
if os.path.isfile(dataset) or dataset in MLLM_DATASET.keys():
- dataset = MLLM_DATASET["liuhaotian/llava"](
+ if seqlen > MLLM_DATASET[dataset].MAX_SUPPORT_SEQLEN:
+ logger.warning(
+ f"seqlen({seqlen}) is greater than the maximum length supported by the {dataset},"
+ f" reset to {MLLM_DATASET[dataset].MAX_SUPPORT_SEQLEN}"
+ )
+ seqlen = 512
+ dataset = MLLM_DATASET[dataset](
template, model, tokenizer, dataset, extra_data_dir, seqlen=seqlen, truncation=truncation, nsamples=nsamples
)
@@ -244,7 +251,7 @@ def get_mllm_dataloader(
set_seed(seed)
dataloader_params = {"batch_size": bs, "shuffle": True, "collate_fn": dataset.template.processor.data_collator}
- return DataLoader(dataset, **dataloader_params), bs, gradient_accumulate_steps
+ return DataLoader(dataset, **dataloader_params), bs, seqlen, gradient_accumulate_steps
else:
# try to load text calibration dataset
from auto_round.calib_dataset import get_dataloader
@@ -256,4 +263,4 @@ def get_mllm_dataloader(
" switching to liuhaotian/llava_conv_58k"
)
exit(-1)
- return dataloader, bs, gradient_accumulate_steps
+ return dataloader, bs, seqlen, gradient_accumulate_steps