Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

9.6 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""MLLM (vision-language) calibration strategy.

Inherits :class:`LLMCalibrator` to reuse ``collect`` / ``cache_inter_data``
and overrides :meth:`calib` to drive the model with multimodal data.

MLLM-specific runtime state (``template`` / ``extra_data_dir`` /
``quant_nontext_module`` / ``template_obj``) lives on the *Compressor*
(populated by ``MLLMMixin.__init__``) and is read here through
``self.compressor``.
"""

import sys

import torch

from auto_round.calibration.llm import LLMCalibrator
from auto_round.calibration.register import register_calibrator
from auto_round.logger import logger
from auto_round.utils import to_device


@register_calibrator("mllm")
class MLLMCalibrator(LLMCalibrator):
    """Calibrator for multimodal (vision-language) models."""

    def __init__(self, compressor):
        super().__init__(compressor)
        self.processor = compressor.processor
        self.image_processor = compressor.image_processor
        self.template = compressor.template
        self.quant_nontext_module = compressor.quant_nontext_module
        self.extra_data_dir = compressor.extra_data_dir

    @torch.no_grad()
    def calib(self, nsamples: int, bs: int) -> None:
        """Drive the multimodal model so block-forward hooks fire.

        Verbatim port of the legacy ``MLLMMixin.calib``.
        """
        from transformers import PreTrainedModel

        from auto_round.compressors.mllm.dataset import get_mllm_dataloader
        from auto_round.compressors.mllm.template import get_template
        from auto_round.special_model_handler import MISTRAL_3_2_MODELS, NOT_SUPPORT_ONLY_TEXT_MODELS
        from auto_round.utils.model import resolve_model_type

        image_processor = self.image_processor
        tokenizer = self.tokenizer
        # Handle template selection
        if isinstance(self.model, PreTrainedModel):
            model_type = getattr(self.model.config, "model_type", None)
            if model_type == "llava" and self.template is None:
                self.template = "default"

        if hasattr(self.model, "name_or_path"):
            name = self.model.name_or_path
            if any(m in name for m in MISTRAL_3_2_MODELS):
                self.template = "mistral3_2"

        template_name = self.template
        if template_name is None:
            template_name = resolve_model_type(self.model) or getattr(self.model.config, "model_type", None)
        if template_name is None:
            template_name = "default"

        template_obj = get_template(
            template_name,
            model=self.model,
            tokenizer=tokenizer,
            processor=self.processor,
            image_processor=image_processor,
            quiet=not self.quant_nontext_module,
        )

        logger.info(f"Using MLLM template: {template_name}")

        from auto_round.utils.dataset_utils import strip_dataset_spec_whitespace

        # Strip only cosmetic whitespace; spaces inside option values
        # (e.g. separator / template / system_prompt) are preserved.
        dataset = strip_dataset_spec_whitespace(self.dataset) if isinstance(self.dataset, str) else self.dataset
        if dataset is None:
            dataset = template_obj.default_dataset

        if isinstance(dataset, str):
            dataset = strip_dataset_spec_whitespace(dataset)
            # Switch text-only dataset to MLLM dataset when quant_nontext_module=True,
            # as text datasets cannot calibrate vision modules.
            from auto_round.calib_dataset import CALIB_DATASETS

            if self.quant_nontext_module and dataset in CALIB_DATASETS:
                logger.warning(
                    "Text only dataset cannot be used for calibrating non-text modules,"
                    " switching to liuhaotian/llava_conv_58k"
                )
                dataset = "liuhaotian/llava_conv_58k"
            elif dataset in CALIB_DATASETS and template_obj.model_type in NOT_SUPPORT_ONLY_TEXT_MODELS:
                logger.warning(
                    f"{getattr(self.model.config, 'model_type', template_obj.model_type)}"
                    f" does not support for {dataset},"
                    " will use liuhaotian/llava_conv_58k with default config as an alternative."
                )
                dataset = "liuhaotian/llava_conv_58k"
            from auto_round.compressors.mllm.dataset import MLLM_DATASET

            if not self.quant_nontext_module and dataset not in MLLM_DATASET:
                # A local text dataset (file/dir/HF
                # text set) is valid calibration for a VLM whose non-text
                # modules are NOT being quantized -- the full-model forward
                # runs text-only and the generic-dict branch of the loop below
                # feeds model(**batch). Upstream instead falls through to
                # get_mllm_dataloader, which KeyErrors on any dataset not in
                # its MLLM_DATASET registry (its `os.path.isfile(dataset) or
                # dataset in MLLM_DATASET` guard still indexes MLLM_DATASET
                # with the file path).
                from auto_round.calib_dataset import get_dataloader

                logger.info(
                    f"Multimodal model with non-MLLM calibration dataset {dataset!r} and "
                    "quant_nontext_module=False: using the standard text dataloader "
                    "(vision/audio towers are not being quantized, so text-only "
                    "calibration through the full-model forward is sufficient)."
                )
                self.dataloader = get_dataloader(
                    tokenizer, self.seqlen, dataset, self.seed, bs, nsamples, trust_remote_code=self.trust_remote_code
                )
            else:
                orig_bs = self.batch_size
                (
                    self.dataloader,
                    self.batch_size,
                    self.seqlen,
                ) = get_mllm_dataloader(
                    template=template_obj,
                    model=self.model,
                    tokenizer=tokenizer,
                    processor=self.processor,
                    image_processor=image_processor,
                    dataset=dataset,
                    extra_data_dir=self.extra_data_dir,
                    seqlen=self.seqlen,
                    bs=bs,
                    seed=self.seed,
                    nsamples=nsamples,
                    quant_nontext_module=self.quant_nontext_module,
                )
                if orig_bs != 1 and self.batch_size == 1:
                    self.is_only_supported_bs1 = True
        else:
            self.dataloader = self.dataset

        # Process data through the model for calibration
        total_cnt = 0
        for data in self.dataloader:
            if data is None:
                continue

            try:
                if isinstance(data, str):
                    # List-of-strings dataset: process through template → model inputs
                    processed = template_obj.processor.get_input(
                        text=data, images=None, max_length=self.seqlen, squeeze=False
                    )
                    data_new = {k: to_device(v, self.model.device) for k, v in processed.items()}
                elif isinstance(data, dict) and "text" in data:
                    # FakeDataLoader-style {"text": ..., "image": ...}: process through template
                    text = data["text"]
                    if isinstance(text, dict):
                        text = [text]
                    input_text = template_obj._encode(text)
                    processed = template_obj.processor.get_input(
                        text=input_text,
                        images=data.get("image", None),
                        max_length=self.seqlen,
                        squeeze=False,
                    )
                    data_new = {}
                    for key, value in processed.items():
                        tensor_val = value if isinstance(value, torch.Tensor) else torch.as_tensor(value)
                        data_new[key] = to_device(tensor_val, self.model.device)
                elif isinstance(data, dict):
                    data_new = {
                        key: value.to(self.model.device) if isinstance(value, torch.Tensor) else value
                        for key, value in data.items()
                    }
                else:
                    data_new = data

                if isinstance(data_new, dict):
                    self.model(**data_new)
                else:
                    self.model(data_new)
            except NotImplementedError:
                pass
            except Exception as e:
                logger.warning(f"Calibration forward pass failed: {e}")
                continue

            total_cnt += bs
            if total_cnt >= nsamples:
                break

        if total_cnt == 0:
            logger.error("no data has been cached, please provide more data")
            sys.exit(-1)
        elif total_cnt < nsamples:
            logger.warning(f"Insufficient number of samples: required {nsamples}, but only {total_cnt} were processed.")