Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

6.2 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""CUDA integration tests for Qwen2.5-Omni and Qwen3-Omni-MoE quantization.

Tests cover end-to-end quantization flow:
- Qwen2.5-Omni: loads real config from pretrained, reduces layers, random weights
  (uses the shared ``tiny_qwen2_5_omni`` session-scoped fixture)
- Qwen3-Omni-MoE: fully synthetic tiny config with random weights
  (no pretrained checkpoint needed — model is too large for CI)
- Quantize with AutoRound
- Save and reload
- Run inference on reloaded model
"""

import os
import shutil
from test.helpers import check_version

import pytest
import torch
from transformers import (
    AutoRoundConfig,
    AutoTokenizer,
    Qwen2_5OmniForConditionalGeneration,
    Qwen3OmniMoeForConditionalGeneration,
)

from auto_round import AutoRound


def _calibration_data(seqlen):
    return [
        {
            "input_ids": torch.ones((1, seqlen), dtype=torch.long),
            "attention_mask": torch.ones((1, seqlen), dtype=torch.long),
        }
    ]


def _make_text_only_autoround(model_path, tokenizer_path, scheme="W4A16"):
    model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(model_path)
    tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
    return AutoRound(
        model,
        tokenizer=tokenizer,
        # Pre-tokenized calibration bypasses processor encoding; reuse the serializable tokenizer for MLLM lifecycle.
        processor=tokenizer,
        dataset=_calibration_data(8),
        scheme=scheme,
        nsamples=1,
        iters=1,
        seqlen=8,
        group_size=32,
        # This smoke uses pre-tokenized text inputs; the Qwen3-Omni template and processor are covered in common tests.
        template="default",
        ignore_layers="self_attn,lm_head,mlp.gate",
    )


def _load_with_torch_backend(save_folder):
    # Small smoke tensors can select a JIT backend; optimized backend selection has dedicated CUDA tests.
    return Qwen3OmniMoeForConditionalGeneration.from_pretrained(
        save_folder, quantization_config=AutoRoundConfig(backend="torch")
    ).to("cuda")


pytestmark = [
    pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"),
    pytest.mark.skipif(
        not check_version("transformers>=5.1.0"),
        reason="Qwen-Omni models require transformers >= 5.1.0",
    ),
]


# ========================= Qwen2.5-Omni Integration Tests ==================


@pytest.mark.skip_ci(reason="Matrix: Not necessary to test all cases in CI")
class TestQwen2_5OmniQuantization:
    """End-to-end quantization test for Qwen2.5-Omni (dense model)."""

    def test_quantize_and_reload(self, tiny_qwen2_5_omni_model_path, tmp_path):
        """Quantize, save, reload, verify weights, and run inference."""
        # Quantize
        autoround = AutoRound(
            tiny_qwen2_5_omni_model_path,
            nsamples=2,
            iters=1,
            seqlen=32,
            ignore_layers="self_attn,lm_head",
        )
        quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=tmp_path)
        assert quantized_model is not None, "Quantized model should not be None"

        # Copy model-specific files required for from_pretrained (e.g. spk_dict.pt for token2wav)
        for extra_file in ["spk_dict.pt"]:
            src = os.path.join(tiny_qwen2_5_omni_model_path, extra_file)
            if os.path.exists(src):
                shutil.copy2(src, save_folder)

        # Reload
        loaded_model = Qwen2_5OmniForConditionalGeneration.from_pretrained(save_folder, device_map="cuda")

        # Run inference on thinker
        inp = torch.randint(0, 100, (1, 64)).to("cuda")
        with torch.inference_mode():
            output = loaded_model.thinker(input_ids=inp)
        assert output is not None, "Inference failed on reloaded model"


# ========================= Qwen3-Omni-MoE Integration Tests ================


class TestQwen3OmniMoeQuantization:
    """End-to-end quantization test for Qwen3-Omni-MoE."""

    @pytest.fixture(autouse=True)
    def _save_dir(self, tmp_path):
        self.save_dir = str(tmp_path / "saved")
        yield
        shutil.rmtree(self.save_dir, ignore_errors=True)

    def test_quantize_and_reload(self, tiny_opt_model_path, tiny_qwen3_omni_moe_model_path):
        """Quantize, save, reload, and run Qwen3-Omni's text-only thinker path."""
        autoround = _make_text_only_autoround(tiny_qwen3_omni_moe_model_path, tiny_opt_model_path)
        quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=self.save_dir)
        assert quantized_model is not None, "Quantized model should not be None"

        loaded_model = _load_with_torch_backend(save_folder)
        inp = torch.randint(0, 100, (1, 8), device="cuda")
        with torch.inference_mode():
            output = loaded_model.thinker(input_ids=inp)
        assert torch.isfinite(output.logits).all(), "Reloaded Qwen3-Omni thinker produced non-finite logits"

    @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all schemes in CI")
    def test_quantize_mxfp4(self, tiny_opt_model_path, tiny_qwen3_omni_moe_model_path):
        """Quantize with MXFP4 scheme and verify."""
        autoround = _make_text_only_autoround(tiny_qwen3_omni_moe_model_path, tiny_opt_model_path, scheme="MXFP4")
        quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=self.save_dir)
        assert quantized_model is not None, "MXFP4 quantized model should not be None"

        # Reload and inference
        loaded_model = _load_with_torch_backend(save_folder)

        inp = torch.randint(0, 100, (1, 64)).to("cuda")
        with torch.inference_mode():
            output = loaded_model.thinker(input_ids=inp)
        assert output is not None