PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""CUDA integration tests for Qwen2.5-Omni and Qwen3-Omni-MoE quantization.
Tests cover end-to-end quantization flow:
- Qwen2.5-Omni: loads real config from pretrained, reduces layers, random weights
(uses the shared ``tiny_qwen2_5_omni`` session-scoped fixture)
- Qwen3-Omni-MoE: fully synthetic tiny config with random weights
(no pretrained checkpoint needed — model is too large for CI)
- Quantize with AutoRound
- Save and reload
- Run inference on reloaded model
"""
import os
import shutil
from test.helpers import check_version
import pytest
import torch
from transformers import (
AutoRoundConfig,
AutoTokenizer,
Qwen2_5OmniForConditionalGeneration,
Qwen3OmniMoeForConditionalGeneration,
)
from auto_round import AutoRound
def _calibration_data(seqlen):
return [
{
"input_ids": torch.ones((1, seqlen), dtype=torch.long),
"attention_mask": torch.ones((1, seqlen), dtype=torch.long),
}
]
def _make_text_only_autoround(model_path, tokenizer_path, scheme="W4A16"):
model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
return AutoRound(
model,
tokenizer=tokenizer,
# Pre-tokenized calibration bypasses processor encoding; reuse the serializable tokenizer for MLLM lifecycle.
processor=tokenizer,
dataset=_calibration_data(8),
scheme=scheme,
nsamples=1,
iters=1,
seqlen=8,
group_size=32,
# This smoke uses pre-tokenized text inputs; the Qwen3-Omni template and processor are covered in common tests.
template="default",
ignore_layers="self_attn,lm_head,mlp.gate",
)
def _load_with_torch_backend(save_folder):
# Small smoke tensors can select a JIT backend; optimized backend selection has dedicated CUDA tests.
return Qwen3OmniMoeForConditionalGeneration.from_pretrained(
save_folder, quantization_config=AutoRoundConfig(backend="torch")
).to("cuda")
pytestmark = [
pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"),
pytest.mark.skipif(
not check_version("transformers>=5.1.0"),
reason="Qwen-Omni models require transformers >= 5.1.0",
),
]
# ========================= Qwen2.5-Omni Integration Tests ==================
@pytest.mark.skip_ci(reason="Matrix: Not necessary to test all cases in CI")
class TestQwen2_5OmniQuantization:
"""End-to-end quantization test for Qwen2.5-Omni (dense model)."""
def test_quantize_and_reload(self, tiny_qwen2_5_omni_model_path, tmp_path):
"""Quantize, save, reload, verify weights, and run inference."""
# Quantize
autoround = AutoRound(
tiny_qwen2_5_omni_model_path,
nsamples=2,
iters=1,
seqlen=32,
ignore_layers="self_attn,lm_head",
)
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=tmp_path)
assert quantized_model is not None, "Quantized model should not be None"
# Copy model-specific files required for from_pretrained (e.g. spk_dict.pt for token2wav)
for extra_file in ["spk_dict.pt"]:
src = os.path.join(tiny_qwen2_5_omni_model_path, extra_file)
if os.path.exists(src):
shutil.copy2(src, save_folder)
# Reload
loaded_model = Qwen2_5OmniForConditionalGeneration.from_pretrained(save_folder, device_map="cuda")
# Run inference on thinker
inp = torch.randint(0, 100, (1, 64)).to("cuda")
with torch.inference_mode():
output = loaded_model.thinker(input_ids=inp)
assert output is not None, "Inference failed on reloaded model"
# ========================= Qwen3-Omni-MoE Integration Tests ================
class TestQwen3OmniMoeQuantization:
"""End-to-end quantization test for Qwen3-Omni-MoE."""
@pytest.fixture(autouse=True)
def _save_dir(self, tmp_path):
self.save_dir = str(tmp_path / "saved")
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
def test_quantize_and_reload(self, tiny_opt_model_path, tiny_qwen3_omni_moe_model_path):
"""Quantize, save, reload, and run Qwen3-Omni's text-only thinker path."""
autoround = _make_text_only_autoround(tiny_qwen3_omni_moe_model_path, tiny_opt_model_path)
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=self.save_dir)
assert quantized_model is not None, "Quantized model should not be None"
loaded_model = _load_with_torch_backend(save_folder)
inp = torch.randint(0, 100, (1, 8), device="cuda")
with torch.inference_mode():
output = loaded_model.thinker(input_ids=inp)
assert torch.isfinite(output.logits).all(), "Reloaded Qwen3-Omni thinker produced non-finite logits"
@pytest.mark.skip_ci(reason="Matrix: Not necessary to test all schemes in CI")
def test_quantize_mxfp4(self, tiny_opt_model_path, tiny_qwen3_omni_moe_model_path):
"""Quantize with MXFP4 scheme and verify."""
autoround = _make_text_only_autoround(tiny_qwen3_omni_moe_model_path, tiny_opt_model_path, scheme="MXFP4")
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=self.save_dir)
assert quantized_model is not None, "MXFP4 quantized model should not be None"
# Reload and inference
loaded_model = _load_with_torch_backend(save_folder)
inp = torch.randint(0, 100, (1, 64)).to("cuda")
with torch.inference_mode():
output = loaded_model.thinker(input_ids=inp)
assert output is not None