Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Fix LLMC CI & Update CPU CI to transformers 5.x (#1604)

Sun, Xuehao1c27c468017f
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
index 70dd09801..398a7da4c 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
@@ -124,7 +124,7 @@ function run_unit_test_llmc() {
rm -rf /root/.venv
uv venv --python=3.12 /root/.venv
uv pip install -U pytest-cov pytest-html
- uv pip install -r test/test_cuda/requirements_llmc.txt
+ BUILD_TYPE="nightly" uv pip install -r test/test_cuda/requirements_llmc.txt --extra-index-url https://download.pytorch.org/whl/cu128 --index-strategy unsafe-best-match
uv pip install .
echo "##[endgroup]"
uv pip list
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.sh
index 2a0134e8c..759f0ed7c 100644
--- a/.azure-pipelines/scripts/ut/run_ut.sh
+++ b/.azure-pipelines/scripts/ut/run_ut.sh
@@ -96,7 +96,7 @@ function run_unit_test() {
}
function run_llmc_unit_test() {
- uv pip install -r /auto-round/test/test_cpu/requirements_llmc.txt
+ BUILD_TYPE="nightly" uv pip install -r /auto-round/test/test_cpu/requirements_llmc.txt
uv pip uninstall auto-round
cd /auto-round && uv pip install .
diff --git a/.azure-pipelines/scripts/ut/run_ut_cuda.sh b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
index aeb236733..b11e17abd 100644
--- a/.azure-pipelines/scripts/ut/run_ut_cuda.sh
+++ b/.azure-pipelines/scripts/ut/run_ut_cuda.sh
@@ -173,7 +173,7 @@ function run_unit_test_llmc() {
cd ${REPO_PATH}/test
rm -rf .coverage* *.xml *.html
- uv pip install -r test_cuda/requirements_llmc.txt
+ BUILD_TYPE="nightly" uv pip install -r test_cuda/requirements_llmc.txt
cd ${REPO_PATH} && uv pip install . && cd ${REPO_PATH}/test
pip list > ${LOG_DIR}/llmc_ut_pip_list.txt
diff --git a/test/test_cpu/advanced/test_low_precision_input_model.py b/test/test_cpu/advanced/test_low_precision_input_model.py
index defee4c37..6d0498b0f 100644
--- a/test/test_cpu/advanced/test_low_precision_input_model.py
+++ b/test/test_cpu/advanced/test_low_precision_input_model.py
@@ -1,5 +1,6 @@
import pytest
import torch
+from packaging import version
from auto_round.utils.weight_handler import (
ModuleWeightType,
@@ -7,7 +8,7 @@ from auto_round.utils.weight_handler import (
convert_module_to_hp_if_necessary,
)
-from ...helpers import get_model_path, get_tiny_model
+from ...helpers import get_model_path, get_tiny_model, transformers_version
class TestCompressedTensor:
@@ -48,6 +49,10 @@ class TestCompressedTensor:
model.model.layers[0].mlp.up_proj.weight.dtype == torch.bfloat16
), "CompressedLinear layer was not converted to Linear"
+ @pytest.mark.skipif(
+ transformers_version >= version.parse("5.0.0"),
+ reason="Compressed-tensor is not compatible with transformers 5.0.0 and above. See https://github.com/vllm-project/compressed-tensors/issues/651",
+ )
def test_mxfp4(self):
model = get_tiny_model(get_model_path(self.mxfp4_model_path))
assert (
diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.py
index e1c38756f..945c28db0 100644
--- a/test/test_cpu/export/test_gguf_format.py
+++ b/test/test_cpu/export/test_gguf_format.py
@@ -263,5 +263,5 @@ class TestGGUF:
gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))
assert gguf_model.get_tensor(2).name == "blk.0.attn_k.weight"
assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"
- assert gguf_model.get_tensor(10).name == "blk.0.ffn_up_exps.weight"
- assert gguf_model.get_tensor(10).tensor_type.name == "Q2_K"
+ assert gguf_model.get_tensor(9).name == "blk.0.ffn_up_exps.weight"
+ assert gguf_model.get_tensor(9).tensor_type.name == "Q2_K"
diff --git a/test/test_cpu/models/test_moe_alignment.py b/test/test_cpu/models/test_moe_alignment.py
index 9b6374f70..feb4eba45 100644
--- a/test/test_cpu/models/test_moe_alignment.py
+++ b/test/test_cpu/models/test_moe_alignment.py
@@ -6,6 +6,7 @@ import torch
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
+from auto_round.modeling.fused_moe import apply_replacements
from auto_round.utils.model import get_module, set_amax_for_all_moe_layers
from ...helpers import get_model_path
@@ -17,11 +18,11 @@ deepseek_v2_lite_path = get_model_path("deepseek-ai/DeepSeek-V2-Lite-Chat")
def setup_deepseek_v2_lite():
"""Fixture to set up the DeepSeek-V2-Lite model for testing."""
model_name = deepseek_v2_lite_path
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=False)
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=False)
# Reduce layers for faster testing
config.num_hidden_layers = 2
- model = AutoModelForCausalLM.from_config(config, trust_remote_code=True)
+ model = AutoModelForCausalLM.from_config(config, trust_remote_code=False)
output_dir = "./tmp/test_moe_alignment_deepseek"
return model, tokenizer, output_dir, config
@@ -109,6 +110,7 @@ def test_set_amax_for_all_moe_layers_direct(setup_deepseek_v2_lite):
os.environ["AR_ENABLE_UNIFY_MOE_INPUT_SCALE"] = "true"
model, tokenizer, output_dir, config = setup_deepseek_v2_lite
+ apply_replacements(model)
# Find the first MoE block and manually set different act_max values
moe_block = None
diff --git a/test/test_cpu/models/test_omni_model.py b/test/test_cpu/models/test_omni_model.py
index ebd1997cf..f69899a12 100644
--- a/test/test_cpu/models/test_omni_model.py
+++ b/test/test_cpu/models/test_omni_model.py
@@ -28,10 +28,7 @@ import shutil
import pytest
import torch
-from transformers import (
- Qwen3OmniMoeConfig,
- Qwen3OmniMoeForConditionalGeneration,
-)
+from transformers import Qwen2_5OmniForConditionalGeneration, Qwen3OmniMoeConfig, Qwen3OmniMoeForConditionalGeneration
from ...helpers import check_version, transformers_version
@@ -87,15 +84,22 @@ def _make_tiny_qwen3_omni_moe_config():
# (real config, reduced layers, random weights). Skipped if model not available.
-class TestQwen2_5OmniBlockNames:
+class TestQwen2_5Omni:
"""Test block name discovery for Qwen2.5-Omni (dense, not MoE)."""
- def test_block_names_default(self, tiny_qwen2_5_omni):
+ @pytest.fixture(autouse=True, scope="class")
+ def setup_and_teardown_class(self, tiny_qwen2_5_omni_model_path, request):
+ request.cls.model = Qwen2_5OmniForConditionalGeneration.from_pretrained(
+ tiny_qwen2_5_omni_model_path, trust_remote_code=True
+ )
+ yield
+ shutil.rmtree("runs", ignore_errors=True)
+
+ def test_block_names_default(self):
"""Test that get_block_names returns thinker + talker layers."""
from auto_round.utils import get_block_names
- model, _, _ = tiny_qwen2_5_omni
- block_names = get_block_names(model, quant_vision=False)
+ block_names = get_block_names(self.model, quant_vision=False)
# Should have thinker.model.layers and talker.model.layers
assert any(
"thinker.model.layers" in str(b) for b in block_names
@@ -104,30 +108,37 @@ class TestQwen2_5OmniBlockNames:
"talker.model.layers" in str(b) for b in block_names
), f"Expected talker.model.layers in block_names, got: {block_names}"
- def test_block_names_quant_vision(self, tiny_qwen2_5_omni):
+ def test_block_names_quant_vision(self):
"""Test that quant_vision adds visual and audio blocks."""
from auto_round.utils import get_block_names
- model, _, _ = tiny_qwen2_5_omni
- blocks_no_vision = get_block_names(model, quant_vision=False)
- blocks_with_vision = get_block_names(model, quant_vision=True)
+ blocks_no_vision = get_block_names(self.model, quant_vision=False)
+ blocks_with_vision = get_block_names(self.model, quant_vision=True)
assert len(blocks_with_vision) > len(blocks_no_vision), "quant_vision=True should add visual/audio blocks"
-
-class TestQwen2_5OmniForward:
- """Test forward function patching for Qwen2.5-Omni."""
-
- def test_handle_special_model(self, tiny_qwen2_5_omni):
+ def test_handle_special_model_forward(self):
"""Test that _handle_special_model patches the forward function."""
from auto_round.special_model_handler import _handle_special_model
# Deepcopy to avoid mutating the shared session-scoped fixture
- model = copy.deepcopy(tiny_qwen2_5_omni[0])
+ model = copy.deepcopy(self.model)
original_forward = model.forward
model = _handle_special_model(model)
assert model.forward != original_forward, "Forward should be patched for qwen2_5_omni"
+ def test_not_moe(self):
+ from auto_round.utils.model import is_moe_layer
+
+ # Thinker uses dense MLP, not MoE
+ thinker_mlp = self.model.thinker.model.layers[0].mlp
+ assert not is_moe_layer(thinker_mlp), "Qwen2.5-Omni should not be detected as MoE"
+
+ def test_not_custom_model(self):
+ from auto_round.modeling.fused_moe.replace_modules import is_custom_model
+
+ assert not is_custom_model(self.model), "Qwen2.5-Omni should not be in BUILTIN_MODULES"
+
class TestQwen2_5OmniProcessor:
"""Test processor and template registration for Qwen2.5-Omni."""
@@ -301,8 +312,22 @@ class TestQwen3OmniMoeReplacement:
new_thinker_out = model.thinker.model.layers[0].mlp(x)
new_talker_out = model.talker.model.layers[0].mlp(x)
- assert torch.allclose(orig_thinker_out, new_thinker_out, atol=1e-5), "Thinker MoE forward mismatch"
- assert torch.allclose(orig_talker_out, new_talker_out, atol=1e-5), "Talker MoE forward mismatch"
+ # Use a NaN-safe comparison: check that NaN positions match, then
+ # compare finite values. This avoids false failures when random
+ # weights cause some outputs to overflow to NaN.
+ for name, orig, new in [
+ ("Thinker", orig_thinker_out, new_thinker_out),
+ ("Talker", orig_talker_out, new_talker_out),
+ ]:
+ assert orig.shape == new.shape, f"{name} shape mismatch: {orig.shape} vs {new.shape}"
+ orig_nan = torch.isnan(orig)
+ new_nan = torch.isnan(new)
+ assert torch.equal(orig_nan, new_nan), f"{name} NaN positions differ"
+ finite_mask = ~orig_nan
+ if finite_mask.any():
+ assert torch.allclose(
+ orig[finite_mask], new[finite_mask], atol=1e-5
+ ), f"{name} MoE forward mismatch on finite values"
class TestQwen3OmniMoeProcessor:
@@ -375,24 +400,6 @@ class TestQwen3OmniMoeUtils:
], f"Expected mlp.gate in ignore_layers for qwen3_omni_moe, got: {ignore_layers}"
-class TestQwen2_5OmniNotMoe:
- """Verify Qwen2.5-Omni is not treated as MoE."""
-
- def test_not_moe(self, tiny_qwen2_5_omni):
- from auto_round.utils.model import is_moe_layer
-
- model, _, _ = tiny_qwen2_5_omni
- # Thinker uses dense MLP, not MoE
- thinker_mlp = model.thinker.model.layers[0].mlp
- assert not is_moe_layer(thinker_mlp), "Qwen2.5-Omni should not be detected as MoE"
-
- def test_not_custom_model(self, tiny_qwen2_5_omni):
- from auto_round.modeling.fused_moe.replace_modules import is_custom_model
-
- model, _, _ = tiny_qwen2_5_omni
- assert not is_custom_model(model), "Qwen2.5-Omni should not be in BUILTIN_MODULES"
-
-
class TestVisualKeysExclusion:
"""Test that omni sub-modules are properly excluded from quantization."""
diff --git a/test/test_cpu/requirements.txt b/test/test_cpu/requirements.txt
index cb1723616..6de266e1d 100644
--- a/test/test_cpu/requirements.txt
+++ b/test/test_cpu/requirements.txt
@@ -6,9 +6,8 @@ torchvision
pillow
numba
pytest
-llmcompressor @ git+https://github.com/vllm-project/llm-compressor.git@main
lm_eval >= 0.4.10 # for transformers >= 5.0.0
diffusers
protobuf
-compressed-tensors==0.14.1a20260313 # temporary pin for llmcompressor
-transformers < 5.0.0
+compressed-tensors
+transformers > 5.0.0
diff --git a/test/test_cpu/requirements_llmc.txt b/test/test_cpu/requirements_llmc.txt
index b6f82d274..a76afdd35 100644
--- a/test/test_cpu/requirements_llmc.txt
+++ b/test/test_cpu/requirements_llmc.txt
@@ -1 +1,2 @@
-llmcompressor @ git+https://github.com/vllm-project/llm-compressor.git@370c04c34191c0e5fb88cf889e16578bf8c7fe20
+llmcompressor @ git+https://github.com/vllm-project/llm-compressor.git@main
+compressed-tensors @ git+https://github.com/vllm-project/compressed-tensors.git@main
diff --git a/test/test_cuda/requirements_llmc.txt b/test/test_cuda/requirements_llmc.txt
index b11e53b7a..50b7bfc9f 100644
--- a/test/test_cuda/requirements_llmc.txt
+++ b/test/test_cuda/requirements_llmc.txt
@@ -1,2 +1,3 @@
llmcompressor @ git+https://github.com/vllm-project/llm-compressor.git@main
+compressed-tensors @ git+https://github.com/vllm-project/compressed-tensors.git@main
parameterized