Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

12.2 KB
Raw Blame
import os
import shutil
from test.helpers import get_model_path, opt_name_or_path

import pytest
from transformers import AutoModelForImageTextToText, AutoProcessor, AutoTokenizer, Qwen2VLForConditionalGeneration

from auto_round import AutoRound
from auto_round.utils import get_block_names


class FakeDataLoader:

    def __init__(self):
        self.batch_size = 1

        self.data = {
            "text": [
                {"role": "user", "content": "<image>\nWhat are the colors of the bus in the image?"},
                {"role": "assistant", "content": "The bus in the image is white and red."},
            ],
            "image": "http://images.cocodataset.org/train2017/000000033471.jpg",
        }

    def __iter__(self):
        for i in range(4):
            yield self.data


class TestAutoRound:

    @classmethod
    def setup_class(self):
        self.model_name = get_model_path("Qwen/Qwen2-VL-2B-Instruct")
        self.dataset = FakeDataLoader()

    @classmethod
    def teardown_class(self):
        shutil.rmtree("runs", ignore_errors=True)

    @pytest.fixture(autouse=True)
    def setup_save_dir(self, tmp_path):
        self.save_dir = str(tmp_path / "saved")

    def test_tune(self, tiny_qwen_vl_model_path):
        bits, group_size = 4, 128
        autoround = AutoRound(
            model=tiny_qwen_vl_model_path,
            bits=bits,
            group_size=group_size,
            nsamples=1,
            batch_size=1,
            iters=2,
            dataset=self.dataset,
            seqlen=10,
        )
        autoround.quantize()
        autoround.save_quantized(self.save_dir, format="auto_gptq", inplace=False)
        autoround.save_quantized(self.save_dir, format="auto_round", inplace=False)

    def test_quant_vision(self, tiny_qwen_vl_model_path):  ## bug need to fix
        tokenizer = AutoTokenizer.from_pretrained(tiny_qwen_vl_model_path)
        processor = AutoProcessor.from_pretrained(tiny_qwen_vl_model_path, trust_remote_code=True)
        model = Qwen2VLForConditionalGeneration.from_pretrained(
            tiny_qwen_vl_model_path, trust_remote_code=True, device_map="auto"
        )
        bits, group_size = 4, 128
        autoround = AutoRound(
            model,
            tokenizer,
            processor=processor,
            bits=bits,
            group_size=group_size,
            nsamples=5,
            batch_size=3,
            iters=2,
            dataset=self.dataset,
            quant_nontext_module=True,
            seqlen=10,
        )
        autoround.quantize()
        autoround.save_quantized(self.save_dir, format="auto_round", inplace=True)

    def test_quant_block_names(self):
        from auto_round.utils import find_matching_blocks

        tokenizer = AutoTokenizer.from_pretrained(self.model_name)
        processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)
        model = Qwen2VLForConditionalGeneration.from_pretrained(
            self.model_name, trust_remote_code=True, device_map="auto"
        )
        to_quant_block_names = "visual.*12,layers.0,model.layers.*9"
        target_blocks = [
            ["model.visual.blocks.12"],
            ["model.language_model.layers.0", "model.language_model.layers.9", "model.language_model.layers.19"],
        ]
        all_blocks = get_block_names(model, quant_vision=True)
        blocks = find_matching_blocks(model, all_blocks, to_quant_block_names)
        assert target_blocks == blocks

    def test_dataset_check(self):
        from auto_round.compressors.mllm.dataset import MLLM_DATASET

        class Myclass:
            model_type = None

        dataset = MLLM_DATASET["liuhaotian/llava"](
            template=Myclass(), model=None, tokenizer=None, dataset_path="liuhaotian/llava", seqlen=32, nsamples=32
        )
        assert len(dataset.questions) == 32
        dataset = MLLM_DATASET["liuhaotian/llava"](
            template=Myclass(), model=None, tokenizer=None, dataset_path="liuhaotian/llava", seqlen=2048, nsamples=512
        )
        assert len(dataset.questions) == 512

    def test_diff_dataset(self, tiny_qwen_vl_model_path):
        tokenizer = AutoTokenizer.from_pretrained(tiny_qwen_vl_model_path)
        processor = AutoProcessor.from_pretrained(tiny_qwen_vl_model_path, trust_remote_code=True)
        model = Qwen2VLForConditionalGeneration.from_pretrained(
            tiny_qwen_vl_model_path, trust_remote_code=True, device_map="auto"
        )
        bits, group_size = 4, 128
        dataset = ["dataset test", "list test"]
        autoround = AutoRound(
            model,
            tokenizer,
            processor=processor,
            bits=bits,
            group_size=group_size,
            nsamples=2,
            batch_size=1,
            iters=2,
            dataset=dataset,
            seqlen=16,
        )
        autoround.quantize()

    def test_pure_text_model_check(self, tiny_qwen_vl_model_path):
        from transformers import AutoModelForCausalLM

        from auto_round.utils import is_pure_text_model

        model = Qwen2VLForConditionalGeneration.from_pretrained(
            tiny_qwen_vl_model_path, trust_remote_code=True, device_map="auto"
        )
        assert not is_pure_text_model(model)
        model = AutoModelForCausalLM.from_pretrained(opt_name_or_path, trust_remote_code=True)
        assert is_pure_text_model(model)

    def test_str_input(self):
        tokenizer = AutoTokenizer.from_pretrained(self.model_name)
        processor = AutoProcessor.from_pretrained(self.model_name, trust_remote_code=True)
        model = Qwen2VLForConditionalGeneration.from_pretrained(
            self.model_name, trust_remote_code=True, device_map="auto"
        )
        bits, group_size = 4, 128
        dataset = ["test pure text", "input for mllm"]
        autoround = AutoRound(
            model,
            tokenizer,
            processor=processor,
            bits=bits,
            group_size=group_size,
            nsamples=2,
            batch_size=1,
            iters=2,
            dataset=dataset,
            seqlen=1,
        )
        autoround.quantize()
        quantized_model_path = self.save_dir
        autoround.save_quantized(quantized_model_path, format="auto_round", inplace=False)
        import requests
        from PIL import Image

        model = Qwen2VLForConditionalGeneration.from_pretrained(
            quantized_model_path,
            torch_dtype="float16",
            device_map="auto",
        )
        processor = AutoProcessor.from_pretrained(quantized_model_path)
        image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
        messages = [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image",
                        "image": image_url,
                    },
                    {"type": "text", "text": "Describe this image."},
                ],
            }
        ]

        # Preparation for inference
        text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        image_inputs = Image.open(requests.get(image_url, stream=True).raw)
        inputs = processor(
            text=[text],
            images=image_inputs,
            padding=True,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)

        generated_ids = model.generate(**inputs, max_new_tokens=128)
        generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]
        output_text = processor.batch_decode(
            generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
        )
        print(output_text[0])

    def test_qwen2_5(self, tiny_qwen_2_5_vl_model_path):
        from auto_round.utils import mllm_load_model

        model_name = tiny_qwen_2_5_vl_model_path
        model, processor, tokenizer, image_processor = mllm_load_model(model_name)
        autoround = AutoRound(
            model,
            tokenizer,
            iters=1,
            nsamples=1,
            seqlen=32,
            quant_nontext_module=True,
            processor=processor,
            image_processor=image_processor,
        )
        _, quantized_model_path = autoround.quantize_and_save(self.save_dir, format="auto_round")

        import requests
        from PIL import Image
        from transformers import AutoProcessor, AutoTokenizer, Qwen2_5_VLForConditionalGeneration

        model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
            quantized_model_path, torch_dtype="auto", device_map="auto"
        )
        block_name_to_quantize = model.config.quantization_config.block_name_to_quantize
        if isinstance(block_name_to_quantize, str):
            block_name_to_quantize = block_name_to_quantize.split(",")
        assert block_name_to_quantize == [
            "model.visual.blocks",
            "model.language_model.layers",
            "model.layers",
        ]
        image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"
        processor = AutoProcessor.from_pretrained(quantized_model_path)
        messages = [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image",
                        "image": image_url,
                    },
                    {"type": "text", "text": "Describe this image."},
                ],
            }
        ]

        text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
        image_inputs = Image.open(requests.get(image_url, stream=True).raw)
        inputs = processor(
            text=[text],
            images=image_inputs,
            padding=True,
            return_tensors="pt",
        )
        inputs = inputs.to(model.device)

        # Inference: Generation of the output
        generated_ids = model.generate(**inputs, max_new_tokens=5)
        generated_ids_trimmed = [out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)]
        output_text = processor.batch_decode(
            generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
        )

    # def test_mllm_early_stop_tracking(self, tiny_qwen_2_5_vl_model_path):
    #     from auto_round.utils import mllm_load_model
    #
    #     model_name = tiny_qwen_2_5_vl_model_path
    #     model, processor, tokenizer, image_processor = mllm_load_model(model_name)
    #     autoround = AutoRound(
    #         model,
    #         tokenizer,
    #         iters=1,
    #         nsamples=1,
    #         seqlen=32,
    #         quant_nontext_module=True,
    #         processor=processor,
    #         image_processor=image_processor,
    #     )
    #
    #     call_log = []
    #     autoround.post_init()
    #     original_should_stop = autoround.calibration._should_stop_cache_forward
    #
    #     def tracked_should_stop(name):
    #         result = original_should_stop(name)
    #         call_log.append(
    #             {
    #                 "name": name,
    #                 "result": result,
    #                 "last_cache_name": getattr(autoround, "last_cache_name", None),
    #             }
    #         )
    #         return result
    #
    #     autoround.calibration._should_stop_cache_forward = tracked_should_stop
    #
    #     try:
    #         all_blocks = get_block_names(model, quant_vision=True)
    #         if not all_blocks:
    #             pytest.skip("No blocks found in the model")
    #
    #         all_first_block_names = [block[0] for block in all_blocks if block]
    #         inputs = autoround.cache_inter_data(all_first_block_names, nsamples=2)
    #         print(f"call_log: {call_log}")
    #         stop_calls = [c for c in call_log if c["result"] is True]
    #         assert len(stop_calls) > 0, "Should trigger early-stop during calibration"
    #         last_cache_values = [c["last_cache_name"] for c in call_log]
    #         assert (
    #             last_cache_values[-1] == "model.language_model.layers.0"
    #         ), "last_cache_name should update to model.language_model.layers.0"
    #         assert "model.language_model.layers.0" in inputs, "Should have cached language model block"
    #     finally:
    #         autoround.calibration._should_stop_cache_forward = original_should_stop