Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

reduce cuda ut runing time (#2266)

Heng Guobdb0d31f65f6
diff --git a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
index 2bfd86e4a..394d2d9cc 100644
--- a/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
+++ b/.azure-pipelines/scripts/cuda_unit_test/run_cuda_ut.sh
@@ -47,6 +47,17 @@ function setup_environment() {
fi
}
+function cleanup_test_workspace() {
+ local tmp_root="/test/tmp"
+ if [ -d "" ]; then
+ find "" -mindepth 1 -maxdepth 1 ! -name tiny_models -exec rm -rf {} +
+ fi
+ rm -rf "/test/ar_work_space"
+ rm -rf "/test/tmp_autoround"
+}
+
+trap cleanup_test_workspace EXIT
+
function print_summary() {
python ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/scripts/ut/print_summary.py --summary-log "${SUMMARY_LOG}"
exit $?
diff --git a/auto_round/calib_dataset.py b/auto_round/calib_dataset.py
index b95e38524..9910b8d24 100644
--- a/auto_round/calib_dataset.py
+++ b/auto_round/calib_dataset.py
@@ -1071,6 +1071,9 @@ def _get_dataset_impl(tokenizer, seqlen, dataset_name="NeelNanda/pile-10k", seed
if len(datasets) > 1:
from datasets import concatenate_datasets
+ # Source metadata is not part of the calibration input and can have
+ # incompatible schemas. Restrict columns only for the merge operation.
+ datasets = [dataset.select_columns(["input_ids", "attention_mask"]) for dataset in datasets]
dataset_final = concatenate_datasets(datasets)
dataset_final = dataset_final.shuffle(seed=seed)
logger.info(dataset_cnt_info)
diff --git a/test/fixtures.py b/test/fixtures.py
index bd793684c..2dfca293e 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -1,3 +1,4 @@
+import json
import os
import shutil
from unittest.mock import patch
@@ -18,10 +19,10 @@ from .helpers import (
get_tiny_model,
gptj_name_or_path,
lamini_name_or_path,
+ make_tiny_qwen3_omni_moe_config,
opt_name_or_path,
phi2_name_or_path,
qwen2_5_omni_name_or_path,
- qwen3_omni_name_or_path,
qwen_2_5_vl_name_or_path,
qwen_moe_name_or_path,
qwen_name_or_path,
@@ -29,60 +30,123 @@ from .helpers import (
save_tiny_model,
)
+_save_tiny_model = save_tiny_model
+TINY_MODEL_ROOT = os.path.join(os.path.dirname(__file__), "tmp", "tiny_models")
+
+
+def tiny_model_dir(name):
+ return os.path.join(TINY_MODEL_ROOT, os.path.basename(os.path.normpath(name)))
+
+
+def _tiny_model_config_fingerprint(path):
+ config_path = os.path.join(path, "config.json")
+ if not os.path.isfile(config_path):
+ return None
+ with open(config_path, encoding="utf-8") as config_file:
+ return json.load(config_file)
+
+
+def _tiny_model_ready(path, signature=None):
+ marker_path = os.path.join(path, ".autoround_ready")
+ if not os.path.isfile(marker_path):
+ return False
+ if signature is None:
+ return True
+ with open(marker_path, encoding="utf-8") as marker:
+ try:
+ metadata = json.load(marker)
+ except json.JSONDecodeError:
+ return False
+ return metadata.get("signature") == signature and metadata.get("config") == _tiny_model_config_fingerprint(path)
+
+
+def _mark_tiny_model(path, signature=None):
+ os.makedirs(path, exist_ok=True)
+ with open(os.path.join(path, ".autoround_ready"), "w", encoding="utf-8") as marker:
+ if signature is None:
+ marker.write("ready\n")
+ else:
+ json.dump({"signature": signature, "config": _tiny_model_config_fingerprint(path)}, marker, sort_keys=True)
+
+
+def _tiny_model_signature(args, kwargs):
+ return json.dumps({"args": args, "kwargs": kwargs}, default=str, sort_keys=True)
+
+
+def save_tiny_model(*args, **kwargs):
+ requested_path = args[1] if len(args) > 1 else kwargs["tiny_model_path"]
+ tiny_model_path = tiny_model_dir(requested_path)
+ signature = _tiny_model_signature(args, kwargs)
+ if _tiny_model_ready(tiny_model_path, signature):
+ return tiny_model_path
+
+ args = list(args)
+ if len(args) > 1:
+ args[1] = tiny_model_path
+ else:
+ kwargs = dict(kwargs)
+ kwargs["tiny_model_path"] = tiny_model_path
+ result = _save_tiny_model(*args, **kwargs)
+ _mark_tiny_model(result, signature)
+ return result
+
# Create tiny model path fixtures for testing
@pytest.fixture(scope="session")
def tiny_opt_model_path():
model_name_or_path = opt_name_or_path
- tiny_model_path = "./tmp/tiny_opt_model_path"
+ tiny_model_path = tiny_model_dir("tiny_opt_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_lamini_model_path():
model_name_or_path = lamini_name_or_path
- tiny_model_path = "./tmp/tiny_lamini_model_path"
+ tiny_model_path = tiny_model_dir("tiny_lamini_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
+
+
[email protected](scope="session")
+def micro_conv1d_model_path():
+ """One-layer Conv1D model for backend-agnostic save/reload smoke tests."""
+ tiny_model_path = tiny_model_dir("micro_conv1d_model_path")
+ tiny_model_path = save_tiny_model(lamini_name_or_path, tiny_model_path, num_layers=1)
+ yield tiny_model_path
@pytest.fixture(scope="session")
def tiny_gptj_model_path():
model_name_or_path = gptj_name_or_path
- tiny_model_path = "./tmp/tiny_gptj_model_path"
+ tiny_model_path = tiny_model_dir("tiny_gptj_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_phi2_model_path():
model_name_or_path = phi2_name_or_path
- tiny_model_path = "./tmp/tiny_phi2_model_path"
+ tiny_model_path = tiny_model_dir("tiny_phi2_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_deepseek_v2_model_path():
model_name_or_path = deepseek_v2_name_or_path
- tiny_model_path = "./tmp/tiny_deepseek_v2_model_path"
+ tiny_model_path = tiny_model_dir("tiny_deepseek_v2_model_path")
tiny_model_path = save_tiny_model(
model_name_or_path, tiny_model_path, num_layers=2, trust_remote_code=False, use_config=True
)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_deepseek_v2_model_path_cpu():
"""Reduced fixture for CPU-only tests (2 MoE layers, 8 experts)."""
model_name_or_path = deepseek_v2_name_or_path
- tiny_model_path = "./tmp/tiny_deepseek_v2_model_path_cpu"
+ tiny_model_path = tiny_model_dir("tiny_deepseek_v2_model_path_cpu")
tiny_model_path = save_tiny_model(
model_name_or_path,
tiny_model_path,
@@ -93,25 +157,22 @@ def tiny_deepseek_v2_model_path_cpu():
config_overrides={"first_k_dense_replace": 0},
)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_gemma_model_path():
model_name_or_path = gemma_name_or_path
- tiny_model_path = "./tmp/tiny_gemma_model_path"
+ tiny_model_path = tiny_model_dir("tiny_gemma_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_qwen_model_path():
model_name_or_path = qwen_name_or_path
- tiny_model_path = "./tmp/tiny_qwen_model_path"
+ tiny_model_path = tiny_model_dir("tiny_qwen_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -120,16 +181,15 @@ def tiny_fp8_qwen_model_path():
with patch("torch.cuda.get_device_capability", return_value=(9, 0)):
model_name_or_path = get_model_path("Qwen/Qwen3-0.6B-FP8")
- tiny_model_path = "./tmp/tiny_fp8_qwen_model_path"
+ tiny_model_path = tiny_model_dir("tiny_fp8_qwen_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_flux_model_path():
model_name_or_path = flux_name_or_path
- tiny_model_path = "./tmp/tiny_flux_model_path"
+ tiny_model_path = tiny_model_dir("tiny_flux_model_path")
tiny_model_path = save_tiny_model(
model_name_or_path,
tiny_model_path,
@@ -147,13 +207,12 @@ def tiny_flux_model_path():
},
)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_z_image_model_path():
model_name_or_path = "Tongyi-MAI/Z-Image"
- tiny_model_path = "./tmp/tiny_z_image_model_path"
+ tiny_model_path = tiny_model_dir("tiny_z_image_model_path")
tiny_model_path = save_tiny_model(
model_name_or_path,
tiny_model_path,
@@ -178,49 +237,47 @@ def tiny_z_image_model_path():
},
)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_untied_qwen_model_path():
model_name_or_path = qwen_name_or_path
- tiny_model_path = "./tmp/tiny_untied_qwen_model_path"
+ tiny_model_path = tiny_model_dir("tiny_untied_qwen_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, force_untie=True)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_qwen_moe_model_path():
model_name_or_path = qwen_moe_name_or_path
- tiny_model_path = "./tmp/tiny_qwen_moe_model_path"
+ tiny_model_path = tiny_model_dir("tiny_qwen_moe_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_qwen_vl_model_path():
model_name_or_path = qwen_vl_name_or_path
- tiny_model_path = "./tmp/tiny_qwen_vl_model_path"
+ tiny_model_path = tiny_model_dir("tiny_qwen_vl_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=3, is_mllm=True)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_qwen_2_5_vl_model_path():
model_name_or_path = qwen_2_5_vl_name_or_path
- tiny_model_path = "./tmp/tiny_qwen_2_5_vl_model_path"
+ tiny_model_path = tiny_model_dir("tiny_qwen_2_5_vl_model_path")
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=2, is_mllm=True)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_fp8_qwen_moe_model_path():
with patch("torch.cuda.get_device_capability", return_value=(9, 0)):
- tiny_model_path = "./tmp/tiny_fp8_qwen_moe_model_path"
+ tiny_model_path = tiny_model_dir("tiny_fp8_qwen_moe_model_path")
+ if _tiny_model_ready(tiny_model_path):
+ yield tiny_model_path
+ return
model_name = get_model_path("Qwen/Qwen3-30B-A3B-FP8")
config = transformers.AutoConfig.from_pretrained(model_name, trust_remote_code=True)
config.num_experts, config.num_hidden_layers, config.vocab_size = 4, 2, 2048
@@ -256,13 +313,16 @@ def tiny_fp8_qwen_moe_model_path():
model.save_pretrained(tiny_model_path)
print(model)
tokenizer.save_pretrained(tiny_model_path)
+ _mark_tiny_model(tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_gpt_oss_model_path():
- tiny_model_path = "./tmp/tiny_gpt_oss"
+ tiny_model_path = tiny_model_dir("tiny_gpt_oss")
+ if _tiny_model_ready(tiny_model_path):
+ yield tiny_model_path
+ return
from transformers import GptOssForCausalLM
model_name = get_model_path("unsloth/gpt-oss-20b")
@@ -274,13 +334,16 @@ def tiny_gpt_oss_model_path():
model.save_pretrained(tiny_model_path)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
tokenizer.save_pretrained(tiny_model_path)
+ _mark_tiny_model(tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_llama4_model_path():
- tiny_model_path = "./tmp/tiny_llama4"
+ tiny_model_path = tiny_model_dir("tiny_llama4")
+ if _tiny_model_ready(tiny_model_path):
+ yield tiny_model_path
+ return
from transformers import Llama4ForConditionalGeneration
model_name = get_model_path("meta-llama/Llama-4-Scout-17B-16E-Instruct")
@@ -302,13 +365,16 @@ def tiny_llama4_model_path():
tokenizer.save_pretrained(tiny_model_path)
processor = transformers.AutoProcessor.from_pretrained(model_name, trust_remote_code=True)
processor.save_pretrained(tiny_model_path)
+ _mark_tiny_model(tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_qwen3_vl_moe_model_path():
- tiny_model_path = "./tmp/tiny_qwen3_vl_moe"
+ tiny_model_path = tiny_model_dir("tiny_qwen3_vl_moe")
+ if _tiny_model_ready(tiny_model_path):
+ yield tiny_model_path
+ return
from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration
model_name = get_model_path("Qwen/Qwen3-VL-30B-A3B-Instruct")
@@ -323,13 +389,16 @@ def tiny_qwen3_vl_moe_model_path():
tokenizer.save_pretrained(tiny_model_path)
processor = transformers.AutoProcessor.from_pretrained(model_name)
processor.save_pretrained(tiny_model_path)
+ _mark_tiny_model(tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_qwen35_moe_model_path():
- tiny_model_path = "./tmp/tiny_qwen35_moe"
+ tiny_model_path = tiny_model_dir("tiny_qwen35_moe")
+ if _tiny_model_ready(tiny_model_path):
+ yield tiny_model_path
+ return
from transformers import Qwen3_5MoeForConditionalGeneration
model_name = get_model_path("Qwen/Qwen3.5-35B-A3B")
@@ -350,13 +419,58 @@ def tiny_qwen35_moe_model_path():
tokenizer.save_pretrained(tiny_model_path)
processor = transformers.AutoProcessor.from_pretrained(model_name)
processor.save_pretrained(tiny_model_path)
+ _mark_tiny_model(tiny_model_path)
+ yield tiny_model_path
+
+
[email protected](scope="session")
+def tiny_qwen35_moe_text_model_path(tiny_opt_model_path):
+ """Small text-only Qwen3.5 MoE fixture for the PR CUDA smoke test."""
+ tiny_model_path = tiny_model_dir("tiny_qwen35_moe_text")
+ if _tiny_model_ready(tiny_model_path):
+ yield tiny_model_path
+ return
+
+ from transformers.models.qwen3_5_moe.configuration_qwen3_5_moe import Qwen3_5MoeTextConfig
+ from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeForCausalLM
+
+ tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_opt_model_path)
+ config = Qwen3_5MoeTextConfig(
+ architectures=["Qwen3_5MoeForCausalLM"],
+ vocab_size=len(tokenizer),
+ hidden_size=128,
+ num_hidden_layers=2,
+ num_attention_heads=4,
+ num_key_value_heads=2,
+ head_dim=32,
+ linear_key_head_dim=32,
+ linear_value_head_dim=32,
+ linear_num_key_heads=4,
+ linear_num_value_heads=4,
+ moe_intermediate_size=64,
+ shared_expert_intermediate_size=64,
+ num_experts=2,
+ num_experts_per_tok=2,
+ layer_types=["linear_attention", "full_attention"],
+ max_position_embeddings=64,
+ use_cache=False,
+ pad_token_id=tokenizer.pad_token_id,
+ bos_token_id=tokenizer.bos_token_id,
+ eos_token_id=tokenizer.eos_token_id,
+ )
+ model = Qwen3_5MoeForCausalLM(config)
+ model.save_pretrained(tiny_model_path)
+ tokenizer.save_pretrained(tiny_model_path)
+ _mark_tiny_model(tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_tiny_llama_model_path():
- tiny_model_path = "./tmp/tiny_TinyLlama"
+ tiny_model_path = tiny_model_dir("tiny_TinyLlama")
+ if _tiny_model_ready(tiny_model_path):
+ yield tiny_model_path
+ return
model_name = get_model_path("TinyLlama/TinyLlama-1.1B-Chat-v1.0")
config = transformers.AutoConfig.from_pretrained(model_name)
config.num_hidden_layers = 4
@@ -364,8 +478,8 @@ def tiny_tiny_llama_model_path():
model.save_pretrained(tiny_model_path)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)
tokenizer.save_pretrained(tiny_model_path)
+ _mark_tiny_model(tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -379,7 +493,10 @@ def tiny_qwen2_5_omni_model_path():
model_name_or_path = get_model_path(qwen2_5_omni_name_or_path)
if not os.path.isdir(model_name_or_path):
pytest.skip("Qwen2.5-Omni fixture is not available locally")
- tiny_model_path = "./tmp/tiny_qwen2_5_omni_model_path"
+ tiny_model_path = tiny_model_dir("tiny_qwen2_5_omni_model_path")
+ if _tiny_model_ready(tiny_model_path):
+ yield tiny_model_path
+ return
tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=1, is_mllm=True, from_config=True)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
processor = transformers.AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True)
@@ -392,26 +509,17 @@ def tiny_qwen2_5_omni_model_path():
else:
pytest.skip("Qwen2.5-Omni spk_dict.pt is not available locally")
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
def tiny_qwen3_omni_moe_model_path():
- """Tiny Qwen3-Omni-MoE model built from real config with reduced layers.
-
- Uses random weights (no checkpoint loading) so it is fast for CI while
- still exercising the real config structure.
- Skipped automatically when the model path does not exist locally.
- """
- model_name_or_path = get_model_path(qwen3_omni_name_or_path)
- tiny_model_path = "./tmp/tiny_qwen3_omni_moe_model_path"
- tiny_model_path = save_tiny_model(model_name_or_path, tiny_model_path, num_layers=1, is_mllm=True, from_config=True)
- tokenizer = transformers.AutoTokenizer.from_pretrained(model_name_or_path, trust_remote_code=True)
- processor = transformers.AutoProcessor.from_pretrained(model_name_or_path, trust_remote_code=True)
- tokenizer.save_pretrained(tiny_model_path)
- processor.save_pretrained(tiny_model_path)
+ """Self-contained tiny Qwen3-Omni-MoE model for CUDA smoke coverage."""
+ tiny_model_path = tiny_model_dir("tiny_qwen3_omni_moe_smoke_v2_model_path")
+ if not _tiny_model_ready(tiny_model_path):
+ model = transformers.Qwen3OmniMoeForConditionalGeneration(make_tiny_qwen3_omni_moe_config())
+ model.save_pretrained(tiny_model_path)
+ _mark_tiny_model(tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
# Mock FP8 capability checks without letting the fake capability affect Inductor code generation.
@@ -428,10 +536,20 @@ def mock_fp8_capable_device():
@pytest.fixture(autouse=True, scope="session")
def clean_tmp_model_folder():
yield
- shutil.rmtree("./tmp", ignore_errors=True) # unittest default workspace
- shutil.rmtree("./ar_work_space", ignore_errors=True) # autoround default workspace
- shutil.rmtree("./tmp_autoround", ignore_errors=True) # autoround default model output path
- # autoround default AutoScheme cache path
+ tmp_root = os.path.join(os.path.dirname(__file__), "tmp")
+ tiny_model_cache = os.path.abspath(TINY_MODEL_ROOT)
+ for entry in os.scandir(tmp_root) if os.path.isdir(tmp_root) else []:
+ if os.path.abspath(entry.path) == tiny_model_cache:
+ continue
+ if entry.is_dir():
+ shutil.rmtree(entry.path, ignore_errors=True)
+ else:
+ try:
+ os.unlink(entry.path)
+ except FileNotFoundError:
+ pass
+ shutil.rmtree("./ar_work_space", ignore_errors=True)
+ shutil.rmtree("./tmp_autoround", ignore_errors=True)
shutil.rmtree(os.path.expanduser("~/.cache/auto_round"), ignore_errors=True)
@@ -489,7 +607,10 @@ def tiny_stable_audio_pipe():
from diffusers.schedulers import EDMDPMSolverMultistepScheduler
from transformers import AutoTokenizer, T5Config, T5EncoderModel
- tiny_model_path = "./tmp/tiny_stable_audio_pipe"
+ tiny_model_path = tiny_model_dir("tiny_stable_audio_pipe")
+ if _tiny_model_ready(tiny_model_path):
+ yield tiny_model_path
+ return
transformer = StableAudioDiTModel(
sample_size=64,
@@ -527,8 +648,8 @@ def tiny_stable_audio_pipe():
scheduler=scheduler,
)
pipe.save_pretrained(tiny_model_path, is_diffusers=True)
+ _mark_tiny_model(tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
@pytest.fixture(scope="session")
@@ -540,7 +661,7 @@ def tiny_mimo_audio_model_path():
Patches config.architectures to ["MiMoAudioModel"] so that resolve_model_type returns 'mimo_audio'.
"""
model_name_or_path = qwen_name_or_path
- tiny_model_path = "./tmp/tiny_mimo_audio_model_path"
+ tiny_model_path = tiny_model_dir("tiny_mimo_audio_model_path")
tiny_model_path = save_tiny_model(
model_name_or_path,
tiny_model_path,
@@ -553,4 +674,130 @@ def tiny_mimo_audio_model_path():
config.architectures = ["MiMoAudioModel"]
config.save_pretrained(tiny_model_path)
yield tiny_model_path
- shutil.rmtree(tiny_model_path, ignore_errors=True)
+
+
+def _micro_model_path(model_name_or_path, name, config_overrides, num_experts=None):
+ """Build a reusable architecture-level model with random micro-sized weights."""
+ model_path = tiny_model_dir(name)
+ if _tiny_model_ready(model_path):
+ config = transformers.AutoConfig.from_pretrained(model_path)
+ if any(getattr(config, key, None) != value for key, value in config_overrides.items()):
+ os.remove(os.path.join(model_path, ".autoround_ready"))
+
+ model_path = save_tiny_model(
+ model_name_or_path,
+ model_path,
+ num_layers=2,
+ from_config=True,
+ num_experts=num_experts,
+ config_overrides=config_overrides,
+ )
+ _save_micro_tokenizer(model_path)
+ return model_path
+
+
+def _save_micro_tokenizer(model_path):
+ from tokenizers import Tokenizer
+ from tokenizers.models import WordLevel
+ from tokenizers.pre_tokenizers import Whitespace
+
+ tokenizer = Tokenizer(
+ WordLevel(
+ {
+ "<pad>": 0,
+ "<bos>": 1,
+ "<eos>": 2,
+ "<unk>": 3,
+ "auto": 4,
+ "round": 5,
+ "calibration": 6,
+ "sample": 7,
+ "keeps": 8,
+ "each": 9,
+ "token": 10,
+ "distinct": 11,
+ "for": 12,
+ "scoring": 13,
+ "another": 14,
+ "local": 15,
+ "dataset": 16,
+ "verifies": 17,
+ "scheme": 18,
+ "behavior": 19,
+ "sequence": 20,
+ },
+ unk_token="<unk>",
+ )
+ )
+ tokenizer.pre_tokenizer = Whitespace()
+ transformers.PreTrainedTokenizerFast(
+ tokenizer_object=tokenizer,
+ bos_token="<bos>",
+ eos_token="<eos>",
+ pad_token="<pad>",
+ unk_token="<unk>",
+ ).save_pretrained(model_path)
+
+
[email protected](scope="session")
+def micro_opt_model_path():
+ yield _micro_model_path(
+ opt_name_or_path,
+ "micro_opt_model_path",
+ {
+ "vocab_size": 256,
+ "hidden_size": 64,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "pad_token_id": 0,
+ "word_embed_proj_dim": 64,
+ "ffn_dim": 128,
+ "num_attention_heads": 4,
+ "max_position_embeddings": 128,
+ },
+ )
+
+
[email protected](scope="session")
+def micro_qwen_model_path():
+ yield _micro_model_path(
+ qwen_name_or_path,
+ "micro_qwen_model_path",
+ {
+ "vocab_size": 32768,
+ "hidden_size": 256,
+ "intermediate_size": 512,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 2,
+ "head_dim": 64,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "pad_token_id": 0,
+ "max_position_embeddings": 128,
+ "tie_word_embeddings": True,
+ },
+ )
+
+
[email protected](scope="session")
+def micro_qwen_moe_model_path():
+ yield _micro_model_path(
+ qwen_moe_name_or_path,
+ "micro_qwen_moe_model_path",
+ {
+ "vocab_size": 256,
+ "hidden_size": 64,
+ "intermediate_size": 128,
+ "moe_intermediate_size": 32,
+ "shared_expert_intermediate_size": 128,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 4,
+ "num_experts_per_tok": 2,
+ "decoder_sparse_step": 1,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "pad_token_id": 0,
+ "max_position_embeddings": 128,
+ },
+ num_experts=4,
+ )
diff --git a/test/helpers.py b/test/helpers.py
index 8bfe6322b..a05b0a9aa 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -173,6 +173,84 @@ qwen3_omni_name_or_path = get_model_path("Qwen/Qwen3-Omni-30B-A3B-Instruct")
flux_name_or_path = get_model_path("black-forest-labs/FLUX.1-dev")
+def make_tiny_qwen3_omni_moe_config():
+ """Build a self-contained Qwen3-Omni-MoE config for unit-test smoke coverage."""
+ config = transformers.Qwen3OmniMoeConfig()
+
+ thinker = config.thinker_config
+ thinker_text = thinker.text_config
+ thinker_text.num_hidden_layers = 1
+ thinker_text.hidden_size = 64
+ thinker_text.intermediate_size = 128
+ thinker_text.moe_intermediate_size = 32
+ thinker_text.num_attention_heads = 4
+ thinker_text.num_key_value_heads = 2
+ thinker_text.num_experts = 4
+ thinker_text.num_experts_per_tok = 2
+ # The text-only forward still reads these multimodal sentinel ids when building positions.
+ thinker.audio_token_id = 151646
+ thinker.image_token_id = 151655
+ thinker.video_token_id = 151656
+ thinker.vision_start_token_id = 151652
+ thinker.audio_start_token_id = 151647
+
+ vision = thinker.vision_config
+ vision.depth = 1
+ vision.hidden_size = 64
+ vision.num_heads = 4
+
+ audio = thinker.audio_config
+ audio.num_hidden_layers = 1
+ audio.d_model = 64
+ audio.encoder_attention_heads = 4
+ audio.encoder_ffn_dim = 128
+ audio.output_dim = 64
+ audio.downsample_hidden_size = 32
+ audio.num_mel_bins = 16
+
+ talker = config.talker_config
+ talker_text = talker.text_config
+ talker_text.num_hidden_layers = 1
+ talker_text.hidden_size = 64
+ talker_text.intermediate_size = 128
+ talker_text.moe_intermediate_size = 32
+ talker_text.num_attention_heads = 4
+ talker_text.num_key_value_heads = 2
+ talker_text.num_experts = 4
+ talker_text.num_local_experts = 4
+ talker_text.num_experts_per_tok = 2
+ talker_text.shared_expert_intermediate_size = 64
+
+ predictor = talker.code_predictor_config
+ predictor.vocab_size = 64
+ predictor.hidden_size = 64
+ predictor.intermediate_size = 128
+ predictor.num_hidden_layers = 1
+ predictor.num_attention_heads = 4
+ predictor.num_key_value_heads = 2
+ predictor.head_dim = 16
+ predictor.num_code_groups = 2
+ predictor.layer_types = ["full_attention"]
+ talker.num_code_groups = 2
+ talker.thinker_hidden_size = 64
+ talker.spatial_merge_size = 2
+
+ code2wav = config.code2wav_config
+ code2wav.codebook_size = 64
+ code2wav.hidden_size = 64
+ code2wav.num_hidden_layers = 1
+ code2wav.num_attention_heads = 4
+ code2wav.num_key_value_heads = 4
+ code2wav.intermediate_size = 128
+ code2wav.num_quantizers = 2
+ code2wav.decoder_dim = 64
+ code2wav.upsample_rates = [2]
+ code2wav.upsampling_ratios = [2]
+
+ config.initializer_range = 0.02
+ return config
+
+
# Slice model into tiny model for speedup
def _reduce_config_layers(config, num_layers, num_experts=None):
"""Reduce num_layers and num_experts in a config object (in-place).
diff --git a/test/integration/test_cuda/test_huggingface.py b/test/integration/test_cuda/test_huggingface.py
index 8aaa95e85..70f2e60a8 100644
--- a/test/integration/test_cuda/test_huggingface.py
+++ b/test/integration/test_cuda/test_huggingface.py
@@ -5,7 +5,7 @@ import pytest
model_name_or_path = "Intel/Qwen3.5-2B-int4-AutoRound"
[email protected]_ci(reason="Requires downloading a large model from HuggingFace")
[email protected]_ci(reason="Architecture: Requires downloading a large model from HuggingFace")
def test_hf():
evaluate_accuracy(
model_name_or_path,
@@ -15,6 +15,6 @@ def test_hf():
)
[email protected]_ci(reason="Requires downloading a large model from HuggingFace")
[email protected]_ci(reason="Architecture: Requires downloading a large model from HuggingFace")
def test_hf_multimodal():
evaluate_accuracy(model_name_or_path, threshold=0.4, batch_size=8, model_type="hf-multimodal", limit=100)
diff --git a/test/integration/test_cuda/test_sglang.py b/test/integration/test_cuda/test_sglang.py
index 8d38df902..a89c0fd6f 100644
--- a/test/integration/test_cuda/test_sglang.py
+++ b/test/integration/test_cuda/test_sglang.py
@@ -207,7 +207,7 @@ class TestAutoRound:
assert "!!!" not in generated_text
- @pytest.mark.skip_ci(reason="Cannot work well in CI env")
+ @pytest.mark.skip_ci(reason="Third-party: Cannot work well in CI env")
def test_awq_format_sglang(self, dataloader):
autoround = AutoRound(
self.model_name,
diff --git a/test/integration/test_cuda/test_vllm.py b/test/integration/test_cuda/test_vllm.py
index c156be31f..8f93e41f4 100644
--- a/test/integration/test_cuda/test_vllm.py
+++ b/test/integration/test_cuda/test_vllm.py
@@ -182,7 +182,7 @@ class TestVllmEvaluation:
not current_platform.is_cuda(),
reason="only supports CUDA backend.",
)
[email protected]_ci(reason="due to vllm and CT Dependency conflict.")
[email protected]_ci(reason="Third-party: due to vllm and CT Dependency conflict.")
def test_vllm_awq_w8a8_llmc_inference(tiny_opt_model_path, tmp_path):
"""W8A8 AWQ → llm_compressor → vLLM: end-to-end inference test."""
save_dir = str(tmp_path / "saved")
diff --git a/test/unit/common/export/test_gguf_dtype_helpers.py b/test/unit/common/export/test_gguf_dtype_helpers.py
index b0bb0353f..71368b8c3 100644
--- a/test/unit/common/export/test_gguf_dtype_helpers.py
+++ b/test/unit/common/export/test_gguf_dtype_helpers.py
@@ -312,3 +312,13 @@ class TestGGUFDTypeSelector:
assert sel.ftype is ftype
assert sel.i_attention_wv == 0
assert sel.i_ffn_down == 0
+
+ def test_q2_k_s_uses_q4_k_for_first_eighth_ffn_down(self):
+ from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector
+
+ selector = GGUFDTypeSelector(
+ {"num_hidden_layers": 8},
+ gguf.LlamaFileType.MOSTLY_Q2_K_S,
+ )
+
+ assert selector.select_qtype("blk.0.ffn_down.weight", n_dims=2) == gguf.GGMLQuantizationType.Q4_K
diff --git a/test/unit/common/inference/test_int_export_backend_selection.py b/test/unit/common/inference/test_int_export_backend_selection.py
new file mode 100644
index 000000000..93a0c66bb
--- /dev/null
+++ b/test/unit/common/inference/test_int_export_backend_selection.py
@@ -0,0 +1,83 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Regression tests for CUDA INT export backend policy without a CUDA runtime.
+
+The export matrix uses the generic Torch backend to cover serialization and
+forward compatibility. Optimized backend functionality is covered by the
+Marlin, ExLlamaV2, and Triton CUDA suites. These tests keep the reference path
+and the representative ``backend=auto`` selection policy separate.
+"""
+
+from pathlib import Path
+
+import pytest
+
+import auto_round.inference.backend as backend_module
+from auto_round.inference.backend import BackendInfos, get_layer_backend
+
+_AUTO_SELECTION_BACKENDS = (
+ "auto_round:torch",
+ "auto_round:tritonv2",
+ "gptqmodel:marlin",
+ "gptqmodel:exllamav2",
+)
+
+
[email protected]("bits", [2, 4, 8])
[email protected]("group_size", [32, 64, 128])
[email protected]("sym", [True, False])
+def test_torch_backend_supports_int_export_matrix(bits, group_size, sym):
+ backend = get_layer_backend(
+ device="cuda",
+ backend="torch",
+ packing_format="auto_round",
+ config={"bits": bits, "group_size": group_size, "sym": sym, "data_type": "int"},
+ in_features=128,
+ out_features=128,
+ )
+
+ assert backend == "auto_round:torch"
+
+
+ "bits,group_size,sym,expected_backend",
+ [
+ (2, 32, True, "auto_round:tritonv2"),
+ (4, 32, True, "gptqmodel:marlin"),
+ (4, 32, False, "gptqmodel:exllamav2"),
+ ],
+)
+def test_auto_backend_selection_for_int_exports(monkeypatch, bits, group_size, sym, expected_backend):
+ """Check capability/priority selection independently of optional packages."""
+ backend_infos = {name: BackendInfos[name] for name in _AUTO_SELECTION_BACKENDS}
+ monkeypatch.setattr(backend_module, "BackendInfos", backend_infos)
+ for backend_info in backend_infos.values():
+ monkeypatch.setattr(backend_info, "requirements", None)
+
+ backend = backend_module.get_layer_backend(
+ device="cuda",
+ backend="auto",
+ packing_format="auto_round",
+ config={"bits": bits, "group_size": group_size, "sym": sym, "data_type": "int"},
+ in_features=128,
+ out_features=128,
+ )
+
+ assert backend == expected_backend
+
+
+def test_generic_int_export_uses_torch_reload_backend():
+ source = (Path(__file__).parents[2] / "test_cuda/export/test_autoround_int_export.py").read_text()
+
+ assert 'AutoRoundConfig(backend="torch")' in source
diff --git a/test/unit/common/models/test_audio_model.py b/test/unit/common/models/test_audio_model.py
index 19524d736..2f0e9a430 100644
--- a/test/unit/common/models/test_audio_model.py
+++ b/test/unit/common/models/test_audio_model.py
@@ -14,14 +14,11 @@
"""Tests for audio model support: MiMo-Audio, Qwen3-TTS, and StableAudio."""
-import os
from types import SimpleNamespace
from unittest.mock import MagicMock
-import pytest
import torch
-from auto_round import AutoRound
from auto_round.compressors.mllm.processor import PROCESSORS
from auto_round.compressors.mllm.template import TEMPLATES
from auto_round.special_model_handler import (
@@ -173,69 +170,6 @@ class TestMiMoAudioForwardPatching:
assert model.forward != original_forward
-class TestMiMoAudioQuantization:
- def test_quantize_rtn(self, tiny_mimo_audio_model_path, tmp_path):
- """RTN-quantizing a tiny MiMo-Audio model produces QuantLinear layers.
-
- Algorithm correctness -- runs once on cpu.
- """
- device = "cpu"
- import transformers
-
- model = transformers.AutoModelForCausalLM.from_pretrained(
- tiny_mimo_audio_model_path, torch_dtype="auto", trust_remote_code=True
- )
- tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_mimo_audio_model_path)
- assert resolve_model_type(model) == "mimo_audio"
-
- autoround = AutoRound(
- model,
- tokenizer=tokenizer,
- scheme="W4A16",
- nsamples=1,
- iters=0,
- seqlen=32,
- device_map=device,
- enable_torch_compile=True,
- disable_opt_rtn=True,
- )
- quantized_model, save_folder = autoround.quantize_and_save(output_dir=str(tmp_path / "saved"))
- assert quantized_model is not None
- assert save_folder is not None
- has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in quantized_model.modules())
- assert has_quantlinear, "Quantized model should contain QuantLinear layers"
-
- @pytest.mark.timeout(90)
- def test_quantize_with_tuning(self, tiny_mimo_audio_model_path, tmp_path):
- """Tuned (iters=1) quantization of a tiny MiMo-Audio model also produces QuantLinear layers.
-
- Algorithm correctness -- runs once on cpu (measured cpu ~21s vs cuda ~18s here,
- not different enough to justify pinning to cuda).
- """
- device = "cpu"
- import transformers
-
- model = transformers.AutoModelForCausalLM.from_pretrained(
- tiny_mimo_audio_model_path, torch_dtype="auto", trust_remote_code=True
- )
- tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_mimo_audio_model_path)
-
- autoround = AutoRound(
- model,
- tokenizer=tokenizer,
- scheme="W4A16",
- nsamples=1,
- iters=1,
- seqlen=32,
- device_map=device,
- enable_torch_compile=True,
- )
- quantized_model, _ = autoround.quantize_and_save(output_dir=str(tmp_path / "saved"))
- assert quantized_model is not None
- has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in quantized_model.modules())
- assert has_quantlinear, "Quantized model should contain QuantLinear layers after tuning"
-
-
# ====================== Qwen3-TTS ======================
@@ -335,40 +269,3 @@ class TestStableAudioPipelineFunction:
type(pipe).__name__ = "FluxPipeline"
_attach_diffusion_pipeline_fn(pipe)
assert not hasattr(pipe, "_autoround_pipeline_fn")
-
-
-class TestStableAudioQuantization:
- def test_quantize_rtn(self, tiny_stable_audio_pipe, tmp_path):
- """RTN-quantizing a tiny StableAudio pipeline saves both quantized and non-quantized components.
-
- Algorithm/export correctness -- runs once on cpu.
- """
- device = "cpu"
- from diffusers import StableAudioPipeline
-
- pipe = StableAudioPipeline.from_pretrained(tiny_stable_audio_pipe)
- output_dir = str(tmp_path / "stable_audio_rtn")
-
- autoround = AutoRound(
- pipe,
- tokenizer=None,
- scheme="W4A16",
- nsamples=1,
- iters=0,
- disable_opt_rtn=True,
- calib_num_inference_steps=2,
- device_map=device,
- enable_torch_compile=True,
- )
- autoround.quantize_and_save(output_dir)
-
- has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in pipe.transformer.modules())
- assert has_quantlinear, "Quantized transformer should contain QuantLinear layers"
- assert os.path.exists(os.path.join(output_dir, "model_index.json")), "model_index.json missing"
- assert os.path.exists(
- os.path.join(output_dir, "transformer", "quantization_config.json")
- ), "quantization_config.json missing in transformer directory"
- # Non-quantized pipeline components must still be saved alongside the quantized one.
- assert os.path.exists(os.path.join(output_dir, "text_encoder")), "text_encoder directory missing"
- assert os.path.exists(os.path.join(output_dir, "vae")), "vae directory missing"
- assert os.path.exists(os.path.join(output_dir, "projection_model")), "projection_model directory missing"
diff --git a/test/unit/common/models/test_conv1d.py b/test/unit/common/models/test_conv1d.py
index 71b4494a3..5a69516d3 100644
--- a/test/unit/common/models/test_conv1d.py
+++ b/test/unit/common/models/test_conv1d.py
@@ -15,7 +15,7 @@
"""Tests for quantizing models built from ``Conv1D`` layers (e.g. GPT-2 style architectures)."""
import shutil
-from test.helpers import get_model_path, get_tiny_model, model_infer
+from test.helpers import model_infer
import pytest
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -43,14 +43,13 @@ class TestQuantizationConv1d:
@pytest.mark.parametrize("device", _AVAILABLE_DEVICES)
@pytest.mark.timeout(300)
- def test_quant(self, dataloader, device):
+ def test_quant(self, dataloader, device, micro_conv1d_model_path):
"""Quantize a Conv1D-based model, save it, reload on `device`, and run inference."""
if device != "cpu" and not is_gptqmodel_available():
pytest.skip("test requires gptqmodel>=2.0")
- model_name = get_model_path("MBZUAI/LaMini-GPT-124M")
- model = get_tiny_model(model_name)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ model = AutoModelForCausalLM.from_pretrained(micro_conv1d_model_path, trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(micro_conv1d_model_path, trust_remote_code=True)
bits, group_size, sym = 4, 128, True
autoround = AutoRound(
@@ -59,7 +58,7 @@ class TestQuantizationConv1d:
bits=bits,
group_size=group_size,
sym=sym,
- iters=2,
+ iters=1,
seqlen=2,
dataset=dataloader,
)
diff --git a/test/unit/common/models/test_omni_model.py b/test/unit/common/models/test_omni_model.py
index 1e916763f..13f349471 100644
--- a/test/unit/common/models/test_omni_model.py
+++ b/test/unit/common/models/test_omni_model.py
@@ -24,11 +24,11 @@ Tests cover:
import copy
import shutil
-from test.helpers import check_version, transformers_version
+from test.helpers import check_version, make_tiny_qwen3_omni_moe_config, transformers_version
import pytest
import torch
-from transformers import Qwen2_5OmniForConditionalGeneration, Qwen3OmniMoeConfig, Qwen3OmniMoeForConditionalGeneration
+from transformers import Qwen2_5OmniForConditionalGeneration, Qwen3OmniMoeForConditionalGeneration
pytestmark = pytest.mark.skipif(
not check_version("transformers>=5.1.0"),
@@ -36,48 +36,6 @@ pytestmark = pytest.mark.skipif(
)
-# ---------------------------------------------------------------------------
-# Helper: create tiny Qwen3-Omni-MoE config (no real checkpoint needed for MoE)
-# ---------------------------------------------------------------------------
-def _make_tiny_qwen3_omni_moe_config():
- config = Qwen3OmniMoeConfig()
- # Thinker
- config.thinker_config.text_config.num_hidden_layers = 1
- config.thinker_config.text_config.hidden_size = 64
- config.thinker_config.text_config.intermediate_size = 128
- config.thinker_config.text_config.moe_intermediate_size = 32
- config.thinker_config.text_config.num_attention_heads = 4
- config.thinker_config.text_config.num_key_value_heads = 2
- config.thinker_config.text_config.num_experts = 4
- config.thinker_config.text_config.num_experts_per_tok = 2
- config.thinker_config.vision_config.depth = 1
- config.thinker_config.vision_config.embed_dim = 64
- config.thinker_config.vision_config.hidden_size = 64
- config.thinker_config.vision_config.num_heads = 4
- config.thinker_config.audio_config.num_hidden_layers = 1
- # Talker
- config.talker_config.text_config.num_hidden_layers = 1
- config.talker_config.text_config.hidden_size = 64
- config.talker_config.text_config.intermediate_size = 128
- config.talker_config.text_config.moe_intermediate_size = 32
- config.talker_config.text_config.num_attention_heads = 4
- config.talker_config.text_config.num_key_value_heads = 2
- config.talker_config.text_config.num_experts = 4
- config.talker_config.text_config.num_local_experts = 4
- config.talker_config.text_config.num_experts_per_tok = 2
- config.talker_config.text_config.shared_expert_intermediate_size = 64
- config.talker_config.thinker_hidden_size = 64
- config.talker_config.spatial_merge_size = 2
- # Code2wav (minimal)
- config.code2wav_config.hidden_size = 64
- config.code2wav_config.num_hidden_layers = 1
- config.code2wav_config.num_attention_heads = 4
- config.code2wav_config.num_key_value_heads = 4
- config.code2wav_config.intermediate_size = 128
- config.initializer_range = 0.02 # Default initializer range for weight initialization
- return config
-
-
# ========================= Qwen2.5-Omni Tests =============================
# NOTE: Tests use the `tiny_qwen2_5_omni` session-scoped fixture from fixtures.py
# (real config, reduced layers, random weights). Skipped if model not available.
@@ -169,7 +127,7 @@ class TestQwen3OmniMoeBlockNames:
"""Test that get_block_names returns thinker layers only by default."""
from auto_round.utils import get_block_names
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
block_names = get_block_names(model, quant_vision=False)
@@ -184,7 +142,7 @@ class TestQwen3OmniMoeBlockNames:
"""Test that quant_vision adds visual and audio blocks."""
from auto_round.utils import get_block_names
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
blocks_no_vision = get_block_names(model, quant_vision=False)
@@ -200,7 +158,7 @@ class TestQwen3OmniMoeForward:
"""Test that _handle_special_model patches the forward function."""
from auto_round.special_model_handler import _handle_special_model
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
original_forward = model.forward
@@ -232,7 +190,7 @@ class TestQwen3OmniMoeReplacement:
"""Test that is_custom_model returns True for Qwen3-Omni-MoE."""
from auto_round.modeling.fused_moe.replace_modules import is_custom_model
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
assert is_custom_model(model)
@@ -240,7 +198,7 @@ class TestQwen3OmniMoeReplacement:
"""Test that only thinker MoE blocks are replaced; talker stays fused."""
from auto_round.modeling.fused_moe.replace_modules import apply_replacements
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
original_talker_class = model.talker.model.layers[0].mlp.__class__.__name__
@@ -276,7 +234,7 @@ class TestQwen3OmniMoeReplacement:
def assert_same_weights(actual: torch.Tensor, expected: torch.Tensor):
torch.testing.assert_close(actual, expected, rtol=0, atol=0, equal_nan=True)
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
# Save original fused thinker weights
@@ -304,7 +262,7 @@ class TestQwen3OmniMoeReplacement:
from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_
torch.manual_seed(0)
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
x = torch.randn(1, 4, 64) * 0.1
@@ -348,7 +306,7 @@ class TestQwen3OmniMoeUtils:
def test_is_moe_layer_thinker(self):
from auto_round.utils.model import is_moe_layer
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
moe_block = model.thinker.model.layers[0].mlp
assert is_moe_layer(moe_block), f"Thinker MoE block ({moe_block.__class__.__name__}) should be detected as MoE"
@@ -356,7 +314,7 @@ class TestQwen3OmniMoeUtils:
def test_is_moe_layer_talker(self):
from auto_round.utils.model import is_moe_layer
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
moe_block = model.talker.model.layers[0].mlp
assert is_moe_layer(moe_block), f"Talker MoE block ({moe_block.__class__.__name__}) should be detected as MoE"
@@ -364,7 +322,7 @@ class TestQwen3OmniMoeUtils:
def test_get_expert_linear_names(self):
from auto_round.utils.model import get_expert_linear_names
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
thinker_mlp = model.thinker.model.layers[0].mlp
@@ -378,7 +336,7 @@ class TestQwen3OmniMoeUtils:
def test_get_expert_input_proj_names(self):
from auto_round.utils.model import get_expert_input_proj_names
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
thinker_mlp = model.thinker.model.layers[0].mlp
@@ -388,7 +346,7 @@ class TestQwen3OmniMoeUtils:
def test_ignore_layers_registered(self):
from auto_round.special_model_handler import get_predefined_ignore_layers
- config = _make_tiny_qwen3_omni_moe_config()
+ config = make_tiny_qwen3_omni_moe_config()
model = Qwen3OmniMoeForConditionalGeneration(config)
ignore_layers = get_predefined_ignore_layers(model)
diff --git a/test/unit/common/quantization/test_asym.py b/test/unit/common/quantization/test_asym.py
index e8b573cec..0f6892057 100644
--- a/test/unit/common/quantization/test_asym.py
+++ b/test/unit/common/quantization/test_asym.py
@@ -240,18 +240,27 @@ class TestAutoRoundAsym:
# ------------------------------------------------------------------
@requires_cuda
@pytest.mark.timeout(120)
- @pytest.mark.parametrize("group_size", [32, 64, 128])
- def test_asym_group_size_tuning(self, tiny_opt_model_path, group_size):
+ @pytest.mark.parametrize("group_size", [32])
+ def test_asym_group_size_tuning(self, tiny_opt_model_path, group_size, dataloader):
"""Tuned (iters=1) asym quantization works across group sizes."""
device = "cuda"
- ar = AutoRound(tiny_opt_model_path, bits=4, group_size=group_size, sym=False, iters=1, seqlen=2, nsamples=1)
+ ar = AutoRound(
+ tiny_opt_model_path,
+ bits=4,
+ group_size=group_size,
+ sym=False,
+ iters=1,
+ seqlen=2,
+ nsamples=1,
+ dataset=dataloader,
+ )
_, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=self.save_dir)
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", device_map=device)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
model_infer(model, tokenizer)
- @pytest.mark.skip_ci(reason="Not necessary since it's covered by backend tests")
+ @pytest.mark.skip_ci(reason="Coverage: Not necessary since it's covered by backend tests")
@pytest.mark.parametrize("device", _device_params(_AVAILABLE_DEVICES))
@pytest.mark.parametrize("bits", [2, 3])
def test_asym_bits_tuning(self, tiny_opt_model_path, bits, device):
@@ -263,7 +272,7 @@ class TestAutoRoundAsym:
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
model_infer(model, tokenizer)
- @pytest.mark.skip_ci(reason="Not necessary since it's covered by backend tests")
+ @pytest.mark.skip_ci(reason="Coverage: Not necessary since it's covered by backend tests")
@pytest.mark.parametrize("device", _device_params(_AVAILABLE_DEVICES))
@pytest.mark.parametrize("format", _FORMATS)
def test_asym_format_tuning(self, tiny_opt_model_path, format, device):
diff --git a/test/unit/common/quantization/test_block_fp.py b/test/unit/common/quantization/test_block_fp.py
index 5ed3b2885..6827520f4 100644
--- a/test/unit/common/quantization/test_block_fp.py
+++ b/test/unit/common/quantization/test_block_fp.py
@@ -1,5 +1,4 @@
import shutil
-import subprocess
from math import ceil
import pytest
@@ -91,46 +90,30 @@ class TestAutoRoundBlockFP:
scale_ref[i, j] = data[i * 128 : (i + 1) * 128, j * 128 : (j + 1) * 128].abs().max() / max_val
assert (scale == scale_ref).all()
- @pytest.mark.timeout(120)
- def test_group_size_handler(self, tiny_qwen_model_path):
- scheme = {
- "data_type": "int",
- "bits": 4,
- "group_size": -1,
- "act_data_type": "int",
- "act_bits": 4,
- "act_group_size": -1,
- }
- autoround = AutoRound(
- tiny_qwen_model_path,
- scheme=scheme,
- iters=2,
- seqlen=2,
- )
+ def test_group_size_handler(self):
+ """The CLI accepts per-channel weight and activation group sizes."""
+ from auto_round.cli.parser import build_quantize_parser
- cmd = [
- "python3",
- "-m",
- "auto_round",
- "--model_name",
- tiny_qwen_model_path,
- "--bits",
- "4",
- "--group_size",
- "-1",
- "--act_bits",
- "4",
- "--act_group_size",
- "-1",
- "--format",
- "fake",
- "--iters",
- "0",
- ]
- subprocess.run(
- cmd,
- check=True,
- text=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
+ args = build_quantize_parser().parse_args(
+ [
+ "--model",
+ "dummy-model",
+ "--bits",
+ "4",
+ "--group_size",
+ "-1",
+ "--act_bits",
+ "4",
+ "--act_group_size",
+ "-1",
+ "--format",
+ "fake",
+ "--iters",
+ "0",
+ ]
)
+
+ assert args.group_size == -1
+ assert args.act_group_size == -1
+ assert args.bits == 4
+ assert args.act_bits == 4
diff --git a/test/unit/common/schemes/test_auto_scheme.py b/test/unit/common/schemes/test_auto_scheme.py
index 5100eb45e..f32ffff4b 100644
--- a/test/unit/common/schemes/test_auto_scheme.py
+++ b/test/unit/common/schemes/test_auto_scheme.py
@@ -16,6 +16,7 @@ from auto_round.auto_scheme.delta_loss import (
_vram_inventory_text,
)
from auto_round.auto_scheme.utils import _build_layer_config_header_rows, _short_summary_name
+from auto_round.export.export_to_gguf.export import _clear_gguf_model_instances
def _make_local_calibration_dataset(tmp_path):
@@ -23,8 +24,8 @@ def _make_local_calibration_dataset(tmp_path):
dataset_path.write_text(
json.dumps(
[
- "auto round calibration sample with enough text for autoscheme scoring. " * 64,
- "another local sample keeps scheme tests independent of network datasets. " * 64,
+ "auto round calibration sample keeps each token distinct for scoring",
+ "another local calibration dataset keeps scoring token sequence distinct",
]
)
)
@@ -295,7 +296,8 @@ def test_build_expert_groups_skips_fixed_layers():
class TestAutoScheme:
@pytest.fixture(autouse=True)
- def setup_save_dir(self, tmp_path):
+ def setup_save_dir(self, tmp_path, monkeypatch):
+ monkeypatch.setenv("AR_AUTO_SCHEME_CACHE", str(tmp_path / "auto_scheme_cache"))
self.save_dir = str(tmp_path / "saved")
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
@@ -307,11 +309,12 @@ class TestAutoScheme:
# Exporting both integer and MXFP schemes loads and rewrites the tiny
# checkpoint twice; on CPU/XPU this can exceed the historical 70s limit.
@pytest.mark.timeout(180)
- def test_auto_scheme_export(self, tiny_opt_model_path):
- model_name = tiny_opt_model_path
+ def test_auto_scheme_export(self, micro_opt_model_path, tmp_path):
+ calibration_dataset = _make_local_calibration_dataset(tmp_path)
+ model_name = micro_opt_model_path
int_save_dir = os.path.join(self.save_dir, "int")
scheme = AutoScheme(avg_bits=2, options=("W2A16"), nsamples=1, ignore_scale_zp_bits=True)
- ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)
+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, seqlen=8, dataset=calibration_dataset)
_, int_model_path = ar.quantize_and_save(int_save_dir)
with open(os.path.join(int_model_path, "config.json")) as f:
@@ -322,7 +325,7 @@ class TestAutoScheme:
mxfp_save_dir = os.path.join(self.save_dir, "mxfp")
scheme = AutoScheme(avg_bits=4, options=("mxfp4"), nsamples=1, ignore_scale_zp_bits=True)
- ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)
+ ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, seqlen=8, dataset=calibration_dataset)
_, mxfp_model_path = ar.quantize_and_save(mxfp_save_dir)
with open(os.path.join(mxfp_model_path, "config.json")) as f:
@@ -333,7 +336,7 @@ class TestAutoScheme:
assert os.path.exists(os.path.join(int_model_path, "config.json"))
@pytest.mark.timeout(120)
- def test_gguf_user_fixed_embedding_budget(self, tiny_qwen_model_path):
+ def test_gguf_user_fixed_embedding_budget(self, micro_qwen_model_path, tmp_path):
"""Regression test: a user-fixed embedding must be budget-priced at its fixed bits.
apply_quant_scheme only covers quant_layer_names (embeddings are carved out),
@@ -341,6 +344,7 @@ class TestAutoScheme:
priced the embedding at 16 bits, making low targets infeasible (DP returned
None -> TypeError, or 'Avg bits is too small').
"""
+ calibration_dataset = _make_local_calibration_dataset(tmp_path)
target_bits = 3.0
scheme = AutoScheme(
avg_bits=target_bits,
@@ -359,12 +363,13 @@ class TestAutoScheme:
}
}
ar = AutoRound(
- model=tiny_qwen_model_path,
+ model=micro_qwen_model_path,
scheme=scheme,
format="gguf:q2_k_s",
iters=0,
nsamples=1,
- seqlen=32,
+ seqlen=8,
+ dataset=calibration_dataset,
layer_config=user_layer_config,
)
weight_numels = {
@@ -372,7 +377,10 @@ class TestAutoScheme:
for n, m in ar.model.named_modules()
if getattr(m, "weight", None) is not None and len(list(m.children())) == 0
}
- model, layer_config = ar.quantize()
+ try:
+ model, layer_config = ar.quantize()
+ finally:
+ _clear_gguf_model_instances()
assert layer_config["model.embed_tokens"]["bits"] == 3
quant_layers = [n for n in layer_config if n in weight_numels]
total_params = sum(weight_numels[n] for n in quant_layers)
@@ -380,14 +388,15 @@ class TestAutoScheme:
avg_bits = total_bits / total_params
assert avg_bits <= target_bits + 0.05
- def test_gguf_embedding_in_budget(self, tiny_qwen_model_path):
+ def test_gguf_embedding_in_budget(self, micro_qwen_model_path, tmp_path):
"""Regression test: the (tied) embedding must be charged against the avg_bits budget.
- On tiny Qwen the embedding holds >90% of the params. Before the fix it was
+ On the micro Qwen fixture the embedding holds >90% of the params. Before the fix it was
silently dropped from the AutoScheme budget (dead `in quant_layer_names`
check) and later filled with the gguf lm_head default (q6_k for tied
embeddings), so the effective avg_bits landed near 6 instead of the target.
"""
+ calibration_dataset = _make_local_calibration_dataset(tmp_path)
target_bits = 3.0
scheme = AutoScheme(
avg_bits=target_bits,
@@ -395,7 +404,15 @@ class TestAutoScheme:
nsamples=1,
ignore_scale_zp_bits=True,
)
- ar = AutoRound(model=tiny_qwen_model_path, scheme=scheme, format="gguf:q2_k_s", iters=0, nsamples=1, seqlen=32)
+ ar = AutoRound(
+ model=micro_qwen_model_path,
+ scheme=scheme,
+ format="gguf:q2_k_s",
+ iters=0,
+ nsamples=1,
+ seqlen=8,
+ dataset=calibration_dataset,
+ )
# Snapshot parameter counts before quantization: gguf packing releases
# weights (module.weight = None) as blocks are packed, so numel is not
# available on the model afterwards.
@@ -404,7 +421,10 @@ class TestAutoScheme:
for n, m in ar.model.named_modules()
if getattr(m, "weight", None) is not None and len(list(m.children())) == 0
}
- model, layer_config = ar.quantize()
+ try:
+ model, layer_config = ar.quantize()
+ finally:
+ _clear_gguf_model_instances()
# Only q2_k_s fits the budget for the embedding (q4_k_s/q6_k alone would exceed it).
assert layer_config["model.embed_tokens"]["bits"] == 2
quant_layers = [n for n in layer_config if n in weight_numels]
@@ -413,7 +433,8 @@ class TestAutoScheme:
avg_bits = total_bits / total_params
assert avg_bits <= target_bits + 0.05
- def test_layer_config(self, tiny_opt_model_path):
+ def test_layer_config(self, tiny_opt_model_path, tmp_path):
+ calibration_dataset = _make_local_calibration_dataset(tmp_path)
from auto_round.auto_scheme.utils import compute_avg_bits_for_model
from auto_round.utils import get_module
@@ -424,7 +445,15 @@ class TestAutoScheme:
# layer-config resolution); the entry still exercises per-layer
# override propagation at a non-pool bit width and group size.
user_layer_config = {"model.decoder.layers.1.fc1": {"bits": 8, "group_size": 32, "sym": True}}
- ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1, layer_config=user_layer_config)
+ ar = AutoRound(
+ model=model_name,
+ scheme=scheme,
+ iters=0,
+ nsamples=1,
+ seqlen=8,
+ dataset=calibration_dataset,
+ layer_config=user_layer_config,
+ )
model, layer_config = ar.quantize()
assert layer_config["model.decoder.layers.1.fc1"]["bits"] == 8
assert layer_config["model.decoder.layers.1.fc1"]["sym"] is True
@@ -435,9 +464,12 @@ class TestAutoScheme:
assert layer.group_size == 32
avg_bits, _ = compute_avg_bits_for_model(model)
print(avg_bits)
- assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3
+ # AutoScheme treats avg_bits as a budget. A fixed 8-bit layer and
+ # discrete candidate schemes can make the closest valid result lower
+ # than the requested target.
+ assert avg_bits <= target_bits + 1e-3
- def test_cache_files_saved_with_correct_format(self, tiny_opt_model_path, tmp_path, monkeypatch):
+ def test_cache_files_saved_with_correct_format(self, micro_opt_model_path, tmp_path, monkeypatch):
"""After AutoScheme runs, per-scheme JSON cache files must exist with individual layer scores."""
import glob
import json
@@ -454,7 +486,7 @@ class TestAutoScheme:
ignore_scale_zp_bits=True,
)
ar = AutoRound(
- model=tiny_opt_model_path,
+ model=micro_opt_model_path,
scheme=scheme,
iters=0,
nsamples=1,
@@ -508,38 +540,22 @@ class TestAutoScheme:
bits_used = {v["bits"] for v in config.values() if "bits" in v}
assert bits_used == {4, 8}, f"expected a mixed MXFP4/MXFP8 config, got {sorted(bits_used)}"
- def test_different_avg_bits_produces_different_layer_config(self, tiny_opt_model_path):
- """Changing avg_bits should change the resulting layer_config."""
- scheme_low = AutoScheme(
- avg_bits=2.5,
- options=("W2A16", "W4A16"),
- nsamples=1,
- ignore_scale_zp_bits=True,
- )
- ar_low = AutoRound(model=tiny_opt_model_path, scheme=scheme_low, iters=0, nsamples=1)
- _, config_low = ar_low.quantize()
+ def test_different_avg_bits_select_different_allocator_paths(self):
+ """A larger feasible budget must allow the lower-loss 4-bit choices."""
+ from auto_round.auto_scheme.delta_loss import choose_bits_per_layer_with_path
- scheme_high = AutoScheme(
- avg_bits=3.5,
- options=("W2A16", "W4A16"),
- nsamples=1,
- ignore_scale_zp_bits=True,
- )
- ar_high = AutoRound(model=tiny_opt_model_path, scheme=scheme_high, iters=0, nsamples=1)
- _, config_high = ar_high.quantize()
+ layers = {
+ "layer.0": [(0, 2, 2.0, ["layer.0"]), (1, 4, 1.0, ["layer.0"])],
+ "layer.1": [(0, 2, 2.0, ["layer.1"]), (1, 4, 1.0, ["layer.1"])],
+ }
- low_avg = sum(v["bits"] for v in config_low.values() if "bits" in v) / max(
- len([v for v in config_low.values() if "bits" in v]), 1
- )
- high_avg = sum(v["bits"] for v in config_high.values() if "bits" in v) / max(
- len([v for v in config_high.values() if "bits" in v]), 1
- )
- assert high_avg > low_avg, (
- f"avg_bits=4 should produce higher average bits than avg_bits=2, "
- f"got low={low_avg:.2f} high={high_avg:.2f}"
- )
+ _, low_budget_path = choose_bits_per_layer_with_path(layers, P=4)
+ _, high_budget_path = choose_bits_per_layer_with_path(layers, P=8)
+
+ assert [scheme for _, scheme in low_budget_path] == [0, 0]
+ assert [scheme for _, scheme in high_budget_path] == [1, 1]
- def test_shared_layers_assigns_same_bits(self, tiny_opt_model_path, tmp_path):
+ def test_shared_layers_assigns_same_bits(self, micro_opt_model_path, tmp_path):
"""With shared_layers=[q_proj,k_proj,v_proj], all three must get the same bits per block."""
scheme = AutoScheme(
avg_bits=5,
@@ -549,7 +565,7 @@ class TestAutoScheme:
shared_layers=[["q_proj", "k_proj", "v_proj"]],
)
ar = AutoRound(
- model=tiny_opt_model_path,
+ model=micro_opt_model_path,
scheme=scheme,
iters=0,
nsamples=1,
diff --git a/test/unit/common/schemes/test_auto_scheme_disk_stream.py b/test/unit/common/schemes/test_auto_scheme_disk_stream.py
index 82c803aa8..44ba567cf 100644
--- a/test/unit/common/schemes/test_auto_scheme_disk_stream.py
+++ b/test/unit/common/schemes/test_auto_scheme_disk_stream.py
@@ -21,6 +21,7 @@ mixed-bit layer_config as the non-streaming baseline, and that the underlying
materialize/free primitives round-trip correctly.
"""
+import json
import os
import shutil
@@ -42,32 +43,47 @@ def _clean_disk_stream_env():
os.environ["AR_DISK_STREAM_MODEL"] = previous
+def _make_local_calibration_dataset(tmp_path):
+ dataset_path = tmp_path / "calibration.json"
+ dataset_path.write_text(
+ json.dumps(
+ [
+ "auto round calibration sample keeps each token distinct for scoring",
+ "another local calibration dataset keeps scoring token sequence distinct",
+ ]
+ )
+ )
+ return str(dataset_path)
+
+
class TestAutoSchemeDiskStream:
@pytest.fixture(autouse=True)
- def setup_save_dir(self, tmp_path):
+ def setup_save_dir(self, tmp_path, monkeypatch):
+ monkeypatch.setenv("AR_AUTO_SCHEME_CACHE", str(tmp_path / "auto_scheme_cache"))
self.save_dir = str(tmp_path / "saved")
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
- def _gen_layer_config(self, model_name, target_bits=3.5):
+ def _gen_layer_config(self, model_name, calibration_dataset, target_bits=3.5):
# iters=1 (the standard tuning loop) rather than iters=0 (RTN): RTN's
# separate block-materialization path doesn't support disk streaming yet
# and is unrelated to this PR, which only streams AutoScheme's own
# sensitivity-scoring pass.
scheme = AutoScheme(avg_bits=target_bits, options=("W2A16", "W4A16", "BF16"), nsamples=1)
- ar = AutoRound(model=model_name, scheme=scheme, iters=1, nsamples=1)
+ ar = AutoRound(model=model_name, scheme=scheme, iters=1, nsamples=1, seqlen=8, dataset=calibration_dataset)
_, layer_config = ar.quantize()
return {name: cfg["bits"] for name, cfg in layer_config.items() if "bits" in cfg}
- def test_disk_stream_matches_baseline_layer_config(self, tiny_opt_model_path):
+ def test_disk_stream_matches_baseline_layer_config(self, micro_opt_model_path, tmp_path):
"""AR_DISK_STREAM_MODEL=1 must select the exact same per-layer bits as the
non-streaming baseline -- streaming changes *how* weights are loaded during
scoring, not the scores themselves."""
+ calibration_dataset = _make_local_calibration_dataset(tmp_path)
os.environ.pop("AR_DISK_STREAM_MODEL", None)
- baseline_bits = self._gen_layer_config(tiny_opt_model_path)
+ baseline_bits = self._gen_layer_config(micro_opt_model_path, calibration_dataset)
os.environ["AR_DISK_STREAM_MODEL"] = "1"
- streamed_bits = self._gen_layer_config(tiny_opt_model_path)
+ streamed_bits = self._gen_layer_config(micro_opt_model_path, calibration_dataset)
assert streamed_bits == baseline_bits
diff --git a/test/unit/common/schemes/test_auto_scheme_low_cpu_mem.py b/test/unit/common/schemes/test_auto_scheme_low_cpu_mem.py
index 721549b2a..da37892a3 100644
--- a/test/unit/common/schemes/test_auto_scheme_low_cpu_mem.py
+++ b/test/unit/common/schemes/test_auto_scheme_low_cpu_mem.py
@@ -19,6 +19,7 @@ This tests the disk offload mechanism for AutoScheme that reduces CPU RAM usage
by offloading block weights to disk during gradient computation.
"""
+import json
import shutil
import pytest
@@ -204,8 +205,19 @@ class TestAutoSchemeIntegration:
"""Integration tests for AutoScheme with low_cpu_mem_usage."""
@pytest.fixture(autouse=True)
- def setup_save_dir(self, tmp_path):
+ def setup_save_dir(self, tmp_path, monkeypatch):
self.save_dir = str(tmp_path / "saved")
+ calibration_path = tmp_path / "calibration.json"
+ calibration_path.write_text(
+ json.dumps(
+ [
+ "auto round calibration sample keeps each token distinct for scoring",
+ "another local calibration dataset keeps scoring token sequence distinct",
+ ]
+ )
+ )
+ self.calibration_dataset = str(calibration_path)
+ monkeypatch.setenv("AR_AUTO_SCHEME_CACHE", str(tmp_path / "auto_scheme_cache"))
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
@@ -213,68 +225,36 @@ class TestAutoSchemeIntegration:
def teardown_class(self):
shutil.rmtree("runs", ignore_errors=True)
- def test_auto_scheme_with_low_cpu_mem_disabled(self, tiny_opt_model_path):
- """Test AutoScheme works normally with low_cpu_mem_usage disabled."""
- model_name = tiny_opt_model_path
- scheme = AutoScheme(
- avg_bits=4,
- options="W2A16,W4A16",
- nsamples=1,
- ignore_scale_zp_bits=True,
- low_cpu_mem_usage=False,
- low_gpu_mem_usage=True,
- )
- ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)
- _, layer_config = ar.quantize()
- assert layer_config is not None
- assert len(layer_config) > 0
-
- def test_auto_scheme_with_low_cpu_mem_enabled(self, tiny_opt_model_path):
- """Test AutoScheme works with low_cpu_mem_usage enabled."""
- model_name = tiny_opt_model_path
- scheme = AutoScheme(
- avg_bits=4,
- options="W2A16,W4A16",
- nsamples=1,
- ignore_scale_zp_bits=True,
- low_cpu_mem_usage=True,
- low_gpu_mem_usage=True,
- )
- ar = AutoRound(model=model_name, scheme=scheme, iters=0, nsamples=1)
- _, layer_config = ar.quantize()
- assert layer_config is not None
- assert len(layer_config) > 0
-
- def test_auto_scheme_low_cpu_mem_results_consistent(self, tiny_opt_model_path):
- """Test that results are consistent with and without low_cpu_mem_usage."""
- model_name = tiny_opt_model_path
-
- # Without low_cpu_mem_usage
- scheme1 = AutoScheme(
- avg_bits=4,
- options="W4A16",
- nsamples=1,
- ignore_scale_zp_bits=True,
- low_cpu_mem_usage=False,
- low_gpu_mem_usage=True,
- )
- ar1 = AutoRound(model=model_name, scheme=scheme1, iters=0, nsamples=1, seed=42)
- _, layer_config1 = ar1.quantize()
-
- # With low_cpu_mem_usage
- scheme2 = AutoScheme(
- avg_bits=4,
- options="W4A16",
- nsamples=1,
- ignore_scale_zp_bits=True,
- low_cpu_mem_usage=True,
- low_gpu_mem_usage=True,
- )
- ar2 = AutoRound(model=model_name, scheme=scheme2, iters=0, nsamples=1, seed=42)
- _, layer_config2 = ar2.quantize()
-
- # Layer configs should have same keys
- assert set(layer_config1.keys()) == set(layer_config2.keys())
+ def test_low_cpu_mem_modes_produce_nonempty_consistent_configs(self, tiny_opt_model_path):
+ """Both offload modes must generate the same non-empty layer-config structure."""
+
+ def quantize_with(low_cpu_mem_usage):
+ scheme = AutoScheme(
+ avg_bits=4,
+ options="W2A16,W4A16",
+ nsamples=1,
+ ignore_scale_zp_bits=True,
+ low_cpu_mem_usage=low_cpu_mem_usage,
+ low_gpu_mem_usage=True,
+ )
+ ar = AutoRound(
+ model=tiny_opt_model_path,
+ scheme=scheme,
+ iters=0,
+ nsamples=1,
+ seqlen=8,
+ dataset=self.calibration_dataset,
+ seed=42,
+ )
+ _, layer_config = ar.quantize()
+ return layer_config
+
+ disabled_config = quantize_with(low_cpu_mem_usage=False)
+ enabled_config = quantize_with(low_cpu_mem_usage=True)
+
+ assert disabled_config
+ assert enabled_config
+ assert set(disabled_config) == set(enabled_config)
class TestOffloadManagerWithModel:
diff --git a/test/unit/common/schemes/test_scheme.py b/test/unit/common/schemes/test_scheme.py
index fe60f1827..1edc23e19 100644
--- a/test/unit/common/schemes/test_scheme.py
+++ b/test/unit/common/schemes/test_scheme.py
@@ -46,13 +46,13 @@ class TestAutoRound:
assert ar.bits == 2
@pytest.mark.timeout(120)
- def test_w4a16_mixed(self, tiny_qwen_moe_model_path, dataloader):
+ def test_w4a16_mixed(self, micro_qwen_moe_model_path, dataloader):
layer_config = {
"model.layers.0.self_attn.k_proj": {"bits": 16},
}
ar = AutoRound(
- tiny_qwen_moe_model_path,
+ micro_qwen_moe_model_path,
scheme="W4A16_MIXED",
nsamples=1,
iters=0,
@@ -213,7 +213,9 @@ class TestAutoRound:
disable_opt_rtn=True,
seqlen=2,
)
- ar.quantize_and_save(self.save_folder)
+ ar.post_init()
+ assert ar.scheme == "GGUF:Q2_K_S"
+ assert ar.scheme_context.bits == 2
def test_fp8_static(self, tiny_opt_model_path):
ar = AutoRound(tiny_opt_model_path, scheme="FP8_STATIC", nsamples=1, iters=1)
diff --git a/test/unit/common/test_micro_models.py b/test/unit/common/test_micro_models.py
new file mode 100644
index 000000000..d21f3fa4b
--- /dev/null
+++ b/test/unit/common/test_micro_models.py
@@ -0,0 +1,72 @@
+from transformers import AutoConfig, AutoTokenizer
+
+
+def test_tiny_model_cache_rejects_a_changed_model_config(tmp_path):
+ from test import fixtures
+
+ signature = fixtures._tiny_model_signature(("model", str(tmp_path)), {"num_layers": 2})
+ config_path = tmp_path / "config.json"
+ config_path.write_text('{"hidden_size": 64}')
+ fixtures._mark_tiny_model(tmp_path, signature)
+
+ config_path.write_text('{"hidden_size": 256}')
+
+ assert not fixtures._tiny_model_ready(tmp_path, signature)
+
+
+def test_tiny_model_cache_does_not_manage_huggingface_source_cache():
+ """Tiny fixture reuse must not delete source checkpoints between pytest runs."""
+ from test import fixtures
+
+ assert not hasattr(fixtures, "_source_model_ids")
+ assert not hasattr(fixtures, "_release_source_model_cache")
+
+
+def test_micro_model_tokenizer_ids_fit_the_model_vocabulary(micro_opt_model_path, micro_qwen_model_path):
+ for model_path in (micro_opt_model_path, micro_qwen_model_path):
+ config = AutoConfig.from_pretrained(model_path)
+ tokenizer = AutoTokenizer.from_pretrained(model_path)
+ input_ids = tokenizer("auto round calibration sample", return_tensors="pt")["input_ids"]
+ special_token_ids = [
+ token_id
+ for token_id in (config.bos_token_id, config.eos_token_id, config.pad_token_id)
+ if token_id is not None
+ ]
+
+ assert input_ids.max().item() < config.vocab_size
+ assert all(token_id < config.vocab_size for token_id in special_token_ids)
+
+
+def test_micro_qwen_preserves_embedding_dominated_parameter_budget(micro_qwen_model_path):
+ from transformers import AutoModelForCausalLM
+
+ model = AutoModelForCausalLM.from_pretrained(micro_qwen_model_path)
+ weights = {
+ name: module.weight.numel()
+ for name, module in model.named_modules()
+ if getattr(module, "weight", None) is not None and not list(module.children())
+ }
+
+ assert (weights["model.embed_tokens"] + weights["lm_head"]) / sum(weights.values()) > 0.9
+
+
+def test_micro_model_tokenizer_keeps_short_calibration_samples_valid(micro_opt_model_path, micro_qwen_model_path):
+ text = "auto round calibration sample keeps each token distinct for scoring"
+ for model_path in (micro_opt_model_path, micro_qwen_model_path):
+ tokenizer = AutoTokenizer.from_pretrained(model_path)
+ input_ids = tokenizer(text)["input_ids"]
+
+ assert len(input_ids) >= 8
+ assert input_ids.count(input_ids[-1]) <= 4
+
+
+def test_micro_qwen_moe_preserves_expert_module_structure(micro_qwen_moe_model_path):
+ from transformers import AutoModelForCausalLM
+
+ model = AutoModelForCausalLM.from_pretrained(micro_qwen_moe_model_path)
+ moe = model.model.layers[0].mlp
+
+ assert model.config.num_experts == 4
+ assert hasattr(moe, "experts")
+ assert hasattr(moe, "shared_expert")
+ assert hasattr(moe.shared_expert, "up_proj")
diff --git a/test/unit/common/test_skip_ci_policy.py b/test/unit/common/test_skip_ci_policy.py
new file mode 100644
index 000000000..a4428b476
--- /dev/null
+++ b/test/unit/common/test_skip_ci_policy.py
@@ -0,0 +1,58 @@
+import ast
+from pathlib import Path
+
+VALID_SKIP_CI_CATEGORIES = {
+ "Accuracy",
+ "Architecture",
+ "Backend/JIT",
+ "Coverage",
+ "Matrix",
+ "Resource",
+ "Third-party",
+}
+
+
+def _skip_ci_reasons(node):
+ reasons = []
+ for decorator in node.decorator_list:
+ if not isinstance(decorator, ast.Call):
+ continue
+ marker = decorator.func
+ if not (
+ isinstance(marker, ast.Attribute)
+ and marker.attr == "skip_ci"
+ and isinstance(marker.value, ast.Attribute)
+ and marker.value.attr == "mark"
+ and isinstance(marker.value.value, ast.Name)
+ and marker.value.value.id == "pytest"
+ ):
+ continue
+ reasons.extend(
+ keyword.value.value
+ for keyword in decorator.keywords
+ if keyword.arg == "reason"
+ and isinstance(keyword.value, ast.Constant)
+ and isinstance(keyword.value.value, str)
+ )
+ return reasons
+
+
+def test_skip_ci_markers_have_one_classified_reason():
+ test_root = Path(__file__).parents[2]
+ violations = []
+ for path in test_root.rglob("test_*.py"):
+ if path == Path(__file__):
+ continue
+ for node in ast.walk(ast.parse(path.read_text())):
+ if not isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)):
+ continue
+ reasons = _skip_ci_reasons(node)
+ if not reasons:
+ continue
+ if len(reasons) != 1:
+ violations.append(f"{path}:{node.lineno} has {len(reasons)} skip_ci markers")
+ continue
+ category, separator, _ = reasons[0].partition(":")
+ if separator != ":" or category not in VALID_SKIP_CI_CATEGORIES:
+ violations.append(f"{path}:{node.lineno} has unclassified reason: {reasons[0]}")
+ assert not violations, "\n".join(violations)
diff --git a/test/unit/common/utils/test_alg_ext.py b/test/unit/common/utils/test_alg_ext.py
index 4926f69fa..7e64bd570 100644
--- a/test/unit/common/utils/test_alg_ext.py
+++ b/test/unit/common/utils/test_alg_ext.py
@@ -1,35 +1,43 @@
-from test.helpers import qwen_name_or_path
+import json
import pytest
from auto_round import AutoRound
-class TestAlgExt:
- @pytest.mark.timeout(60)
- def test_alg_ext(self, tiny_opt_model_path, tiny_qwen_model_path):
- model_name = tiny_opt_model_path
- ar = AutoRound(model_name, scheme="W2A16", iters=1, nsamples=1, enable_alg_ext=True)
- ar.quantize()
-
- model_name = tiny_qwen_model_path
- ar = AutoRound(model_name, scheme="gguf:q4_k_s", iters=1, nsamples=1, enable_alg_ext=True)
- ar.quantize()
+def _local_calibration_dataset(tmp_path):
+ dataset_path = tmp_path / "alg_ext_calibration.json"
+ dataset_path.write_text(json.dumps(["Algorithm extension local calibration sample. " * 32]))
+ return str(dataset_path)
- from auto_round.auto_scheme import AutoScheme
- scheme = AutoScheme(options=["mxfp4", "mxfp8"], avg_bits=5.5, ignore_scale_zp_bits=True)
- model_name = tiny_qwen_model_path
- ar = AutoRound(model_name, scheme=scheme, iters=1, nsamples=1, enable_alg_ext=True, enable_torch_compile=True)
- ar.quantize()
+class TestAlgExt:
+ @pytest.mark.timeout(60)
+ def test_gguf_q4_alg_ext(self, tiny_qwen_model_path, tmp_path):
+ """Exercise the GGUF algorithm-extension path not covered by CUDA matrix tests."""
+ AutoRound(
+ tiny_qwen_model_path,
+ scheme="gguf:q4_k_s",
+ iters=1,
+ nsamples=1,
+ seqlen=32,
+ dataset=_local_calibration_dataset(tmp_path),
+ enable_alg_ext=True,
+ ).quantize()
def test_alg_ext_import(self):
from auto_round.algorithms.quantization.sign_roundv2 import SignRoundV2Quantizer
- def test_all_support_dtype(self, tiny_opt_model_path):
- model_name = tiny_opt_model_path
- for scheme in ["MXFP4", "NVFP4", "W2A16G64"]:
- ar = AutoRound(
- model_name, scheme=scheme, iters=1, nsamples=1, enable_alg_ext=True, enable_torch_compile=True
- )
- ar.quantize()
+ @pytest.mark.timeout(60)
+ def test_nvfp4_alg_ext(self, tiny_opt_model_path, tmp_path):
+ """Keep NVFP4 as the representative format not exercised by the CUDA matrix."""
+ AutoRound(
+ tiny_opt_model_path,
+ scheme="NVFP4",
+ iters=1,
+ nsamples=1,
+ seqlen=32,
+ dataset=_local_calibration_dataset(tmp_path),
+ enable_alg_ext=True,
+ enable_torch_compile=True,
+ ).quantize()
diff --git a/test/unit/common/utils/test_calib_dataset_helpers.py b/test/unit/common/utils/test_calib_dataset_helpers.py
index cc2b78ada..c21088143 100644
--- a/test/unit/common/utils/test_calib_dataset_helpers.py
+++ b/test/unit/common/utils/test_calib_dataset_helpers.py
@@ -16,6 +16,7 @@
from unittest.mock import MagicMock
import pytest
+from datasets import Dataset, Features, Sequence, Value
# ---------------------------------------------------------------------------
@@ -292,3 +293,68 @@ class TestApplyChatTemplateToSamples:
second_call_args = tokenizer.apply_chat_template.call_args_list[1]
msgs_arg = second_call_args[0][0]
assert all(m["role"] != "system" for m in msgs_arg)
+
+
+# ---------------------------------------------------------------------------
+# _get_dataset_impl
+# ---------------------------------------------------------------------------
+class TestGetDatasetImpl:
+ def test_combines_sources_with_different_metadata_schemas(self, monkeypatch):
+ import auto_round.calib_dataset as calib_dataset
+
+ def dataset_loader(text_feature):
+ def load_dataset(*args, **kwargs):
+ return Dataset.from_dict(
+ {
+ "text": ["calibration sample"],
+ "input_ids": [list(range(8))],
+ "attention_mask": [[1] * 8],
+ },
+ features=Features(
+ {
+ "text": Value(text_feature),
+ "input_ids": Sequence(Value("int64")),
+ "attention_mask": Sequence(Value("int8")),
+ }
+ ),
+ )
+
+ return load_dataset
+
+ monkeypatch.setattr(
+ calib_dataset,
+ "CALIB_DATASETS",
+ {
+ "string_metadata": dataset_loader("string"),
+ "large_string_metadata": dataset_loader("large_string"),
+ },
+ )
+
+ dataset = calib_dataset._get_dataset_impl(
+ tokenizer=MagicMock(),
+ seqlen=8,
+ dataset_name="string_metadata,large_string_metadata",
+ nsamples=2,
+ )
+
+ assert dataset.column_names == ["input_ids", "attention_mask"]
+ assert len(dataset) == 2
+
+ def test_preserves_single_source_metadata(self, monkeypatch):
+ import auto_round.calib_dataset as calib_dataset
+
+ def load_dataset(*args, **kwargs):
+ return Dataset.from_dict(
+ {
+ "source": ["unit-test"],
+ "input_ids": [list(range(8))],
+ "attention_mask": [[1] * 8],
+ }
+ )
+
+ monkeypatch.setattr(calib_dataset, "CALIB_DATASETS", {"single_source": load_dataset})
+ dataset = calib_dataset._get_dataset_impl(
+ tokenizer=MagicMock(), seqlen=8, dataset_name="single_source", nsamples=1
+ )
+
+ assert dataset.column_names == ["source", "input_ids", "attention_mask"]
diff --git a/test/unit/common/utils/test_cli_usage.py b/test/unit/common/utils/test_cli_usage.py
index fbc0f2935..e25234641 100644
--- a/test/unit/common/utils/test_cli_usage.py
+++ b/test/unit/common/utils/test_cli_usage.py
@@ -1,5 +1,6 @@
import argparse
import inspect
+import json
import shutil
import sys
from test.helpers import get_model_path
@@ -41,7 +42,10 @@ class TestAutoRoundCmd:
_assert_cli_ok(monkeypatch, ["auto_round", "-h"])
@pytest.mark.timeout(90)
- def test_auto_round_cmd2(self, monkeypatch, tiny_opt_model_path):
+ def test_auto_round_cmd2(self, monkeypatch, tiny_opt_model_path, tmp_path):
+ calibration_path = tmp_path / "calibration.json"
+ calibration_path.write_text(json.dumps(["AutoRound CLI local calibration sample. " * 32]))
+
_assert_cli_ok(
monkeypatch,
[
@@ -54,14 +58,12 @@ class TestAutoRoundCmd:
"2",
"--nsamples",
"1",
+ "--dataset",
+ str(calibration_path),
"--format",
"auto_gptq,auto_round",
"--output_dir",
self.save_dir,
- "--tasks",
- "piqa",
- "--limit",
- "2",
],
)
@@ -108,27 +110,19 @@ class TestAutoRoundCmd:
"2",
]
- @pytest.mark.timeout(90)
- def test_auto_round_cmd4(self, monkeypatch):
- _assert_cli_ok(
- monkeypatch,
- [
- "auto_round",
- "--seqlen",
- "8",
- "--iter",
- "2",
- "--nsamples",
- "8",
- "--output_dir",
- self.save_dir,
- "--tasks",
- "lambada_openai",
- "--limit",
- "2",
- ],
- entry=run_light,
- )
+ def test_auto_round_cmd4_applies_light_recipe(self, monkeypatch):
+ from auto_round.cli import main as cli_main
+
+ captured = {}
+ monkeypatch.setattr(cli_main, "tune", lambda args: captured.setdefault("args", args))
+ monkeypatch.setattr(sys, "argv", ["auto-round-light", "--model", "dummy-model"])
+
+ run_light()
+
+ args = captured["args"]
+ assert args.iters == 50
+ assert args.seqlen == 2048
+ assert args.nsamples == 128
def test_auto_round_cmd5(self, monkeypatch):
_assert_cli_ok(monkeypatch, ["auto_round", "--eval", "-h"])
@@ -136,15 +130,19 @@ class TestAutoRoundCmd:
def test_auto_round_cmd6(self, monkeypatch):
_assert_cli_ok(monkeypatch, ["auto_round", "--eval", "--lmms", "-h"])
- @pytest.mark.timeout(90)
- def test_auto_round_cmd7(self, monkeypatch, tiny_qwen_vl_model_path):
- _assert_cli_ok(
- monkeypatch,
+ def test_auto_round_cmd7_routes_mllm_without_quantizing(self, monkeypatch):
+ from auto_round.cli import main as cli_main
+
+ captured = {}
+ monkeypatch.setattr(cli_main, "tune", lambda args: captured.setdefault("args", args))
+ monkeypatch.setattr(
+ sys,
+ "argv",
[
"auto_round",
"--mllm",
"--model",
- tiny_qwen_vl_model_path,
+ "dummy-model",
"--iter",
"2",
"--nsamples",
@@ -153,11 +151,16 @@ class TestAutoRoundCmd:
"32",
"--format",
"auto_round",
- "--output_dir",
- self.save_dir,
],
)
+ run()
+
+ args = captured["args"]
+ assert args.model_name == "dummy-model"
+ assert args.mllm is True
+ assert args.iters == 2
+
def test_auto_round_cmd8_routes_quant_nontext_module_without_quantizing(self, monkeypatch):
from auto_round.cli import main as cli_main
diff --git a/test/unit/test_cpu/backends/test_torch_backend.py b/test/unit/test_cpu/backends/test_torch_backend.py
index f8af431fc..5ade5afe2 100644
--- a/test/unit/test_cpu/backends/test_torch_backend.py
+++ b/test/unit/test_cpu/backends/test_torch_backend.py
@@ -82,7 +82,9 @@ class TestTorchBackendFunctional:
tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
model_infer(model, tokenizer)
- @pytest.mark.skip_ci(reason="Not necessary to test both symmetric and asymmetric for 3-bit quantization in CI")
+ @pytest.mark.skip_ci(
+ reason="Matrix: Not necessary to test both symmetric and asymmetric for 3-bit quantization in CI"
+ )
@pytest.mark.parametrize("device", _AVAILABLE_DEVICES)
def test_torch_backend_3bit_sym(self, tiny_opt_model_path, dataloader, device):
"""3-bit symmetric model loads via the torch backend and generates text."""
diff --git a/test/unit/test_cpu/models/test_audio_quantization.py b/test/unit/test_cpu/models/test_audio_quantization.py
new file mode 100644
index 000000000..faa693b5d
--- /dev/null
+++ b/test/unit/test_cpu/models/test_audio_quantization.py
@@ -0,0 +1,124 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""CPU-only quantization coverage for audio model architectures."""
+
+import os
+
+import pytest
+import torch
+
+from auto_round import AutoRound
+from auto_round.special_model_handler import resolve_model_type
+
+
+class _LocalCalibrationDataLoader:
+ """Provide deterministic token batches without downloading a calibration dataset."""
+
+ def __iter__(self):
+ yield torch.ones((1, 32), dtype=torch.long)
+
+
+class TestMiMoAudioQuantization:
+ def test_quantize_rtn(self, tiny_mimo_audio_model_path, tmp_path):
+ """RTN-quantizing a tiny MiMo-Audio model produces QuantLinear layers.
+
+ Algorithm correctness -- runs once on cpu.
+ """
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ tiny_mimo_audio_model_path, torch_dtype="auto", trust_remote_code=True
+ )
+ tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_mimo_audio_model_path)
+ assert resolve_model_type(model) == "mimo_audio"
+
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ scheme="W4A16",
+ nsamples=1,
+ iters=0,
+ seqlen=32,
+ device_map="cpu",
+ enable_torch_compile=True,
+ disable_opt_rtn=True,
+ )
+ quantized_model, save_folder = autoround.quantize_and_save(output_dir=str(tmp_path / "saved"))
+ assert quantized_model is not None
+ assert save_folder is not None
+ has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in quantized_model.modules())
+ assert has_quantlinear, "Quantized model should contain QuantLinear layers"
+
+ @pytest.mark.timeout(90)
+ def test_quantize_with_tuning(self, tiny_mimo_audio_model_path, tmp_path):
+ """Tuned MiMo-Audio quantization uses local calibration data and produces QuantLinear layers."""
+ import transformers
+
+ model = transformers.AutoModelForCausalLM.from_pretrained(
+ tiny_mimo_audio_model_path, torch_dtype="auto", trust_remote_code=True
+ )
+ tokenizer = transformers.AutoTokenizer.from_pretrained(tiny_mimo_audio_model_path)
+
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ scheme="W4A16",
+ nsamples=1,
+ iters=1,
+ seqlen=32,
+ dataset=_LocalCalibrationDataLoader(),
+ device_map="cpu",
+ enable_torch_compile=True,
+ )
+ quantized_model, _ = autoround.quantize_and_save(output_dir=str(tmp_path / "saved"))
+ assert quantized_model is not None
+ has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in quantized_model.modules())
+ assert has_quantlinear, "Quantized model should contain QuantLinear layers after tuning"
+
+
+class TestStableAudioQuantization:
+ def test_quantize_rtn(self, tiny_stable_audio_pipe, tmp_path):
+ """RTN-quantizing a tiny StableAudio pipeline saves both quantized and non-quantized components.
+
+ Algorithm/export correctness -- runs once on cpu.
+ """
+ from diffusers import StableAudioPipeline
+
+ pipe = StableAudioPipeline.from_pretrained(tiny_stable_audio_pipe)
+ output_dir = str(tmp_path / "stable_audio_rtn")
+
+ autoround = AutoRound(
+ pipe,
+ tokenizer=None,
+ scheme="W4A16",
+ nsamples=1,
+ iters=0,
+ disable_opt_rtn=True,
+ calib_num_inference_steps=2,
+ device_map="cpu",
+ enable_torch_compile=True,
+ )
+ autoround.quantize_and_save(output_dir)
+
+ has_quantlinear = any(m.__class__.__name__ == "QuantLinear" for m in pipe.transformer.modules())
+ assert has_quantlinear, "Quantized transformer should contain QuantLinear layers"
+ assert os.path.exists(os.path.join(output_dir, "model_index.json")), "model_index.json missing"
+ assert os.path.exists(
+ os.path.join(output_dir, "transformer", "quantization_config.json")
+ ), "quantization_config.json missing in transformer directory"
+ # Non-quantized pipeline components must still be saved alongside the quantized one.
+ assert os.path.exists(os.path.join(output_dir, "text_encoder")), "text_encoder directory missing"
+ assert os.path.exists(os.path.join(output_dir, "vae")), "vae directory missing"
+ assert os.path.exists(os.path.join(output_dir, "projection_model")), "projection_model directory missing"
diff --git a/test/unit/common/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.py
similarity index 97%
rename from test/unit/common/quantization/test_model_free.py
rename to test/unit/test_cpu/quantization/test_model_free.py
index 6f082b450..2263e07a6 100644
--- a/test/unit/common/quantization/test_model_free.py
+++ b/test/unit/test_cpu/quantization/test_model_free.py
@@ -12,7 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Unit tests for auto_round.compressors.model_free module."""
+"""Unit tests for auto_round.compressors.model_free module.
+
+These generic model-free configuration and export tests are deliberately CPU-only.
+Backend-specific model-free behavior is covered by test_model_free_parity.py.
+"""
import json
import os
@@ -646,7 +650,7 @@ class TestModelFreeQuantize:
def test_basic(self, tmp_path):
model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)
output_dir = str(tmp_path / "output")
- AutoRound(model=model_dir, scheme="W4A16", model_free=True).quantize_and_save(output_dir)
+ AutoRound(model=model_dir, scheme="W4A16", model_free=True, device_map="cpu").quantize_and_save(output_dir)
qc = _read_qconfig(output_dir)
assert qc["quant_method"] == "auto-round" and qc["bits"] == 4 and qc["model_free"] is True
keys = _read_output_keys(output_dir)
@@ -660,7 +664,9 @@ class TestModelFreeQuantize:
}
model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
output_dir = str(tmp_path / "output")
- AutoRound(model=model_dir, scheme="W4A16", model_free=True, ignore_layers="mlp").quantize_and_save(output_dir)
+ AutoRound(
+ model=model_dir, scheme="W4A16", model_free=True, ignore_layers="mlp", device_map="cpu"
+ ).quantize_and_save(output_dir)
keys = _read_output_keys(output_dir)
assert "model.layers.0.mlp.fc1.weight" in keys and "model.layers.0.mlp.fc1.qweight" not in keys
assert "model.layers.0.self_attn.q_proj.qweight" in keys
@@ -668,13 +674,15 @@ class TestModelFreeQuantize:
def test_multi_shard(self, tmp_path):
model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS, multi_shard=True)
output_dir = str(tmp_path / "output")
- AutoRound(model=model_dir, scheme="W4A16", model_free=True).quantize_and_save(output_dir)
+ AutoRound(model=model_dir, scheme="W4A16", model_free=True, device_map="cpu").quantize_and_save(output_dir)
assert os.path.exists(os.path.join(output_dir, "model.safetensors.index.json"))
def test_quant_lm_head(self, tmp_path):
model_dir = _make_model_dir(tmp_path, _SIMPLE_CONFIG, _SIMPLE_TENSORS)
output_dir = str(tmp_path / "output")
- AutoRound(model=model_dir, scheme="W4A16", model_free=True, quant_lm_head=True).quantize_and_save(output_dir)
+ AutoRound(
+ model=model_dir, scheme="W4A16", model_free=True, quant_lm_head=True, device_map="cpu"
+ ).quantize_and_save(output_dir)
assert "lm_head.qweight" in _read_output_keys(output_dir)
def test_layer_config_lm_head_bits_takes_effect(self, tmp_path):
@@ -686,6 +694,7 @@ class TestModelFreeQuantize:
scheme="W2A16G64",
model_free=True,
layer_config={"lm_head": {"bits": 4}},
+ device_map="cpu",
).quantize_and_save(output_dir)
assert "lm_head.qweight" in _read_output_keys(output_dir)
@@ -698,6 +707,7 @@ class TestModelFreeQuantize:
scheme="W2A16G64",
model_free=True,
layer_config={"lm_head": {"scheme": "W4A16"}},
+ device_map="cpu",
).quantize_and_save(output_dir)
assert "lm_head.qweight" in _read_output_keys(output_dir)
@@ -705,7 +715,10 @@ class TestModelFreeQuantize:
model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})
output_dir = str(tmp_path / "output")
AutoRound(
- model=model_dir, scheme=QuantizationScheme(bits=4, group_size=64, sym=False), model_free=True
+ model=model_dir,
+ scheme=QuantizationScheme(bits=4, group_size=64, sym=False),
+ model_free=True,
+ device_map="cpu",
).quantize_and_save(output_dir)
qc = _read_qconfig(output_dir)
assert qc["sym"] is False and qc["group_size"] == 64
@@ -985,7 +998,7 @@ class TestLLMCompressorMXFPSource:
}
model_dir = _make_model_dir(tmp_path, _DEEPSEEK_V4_CFG, tensors)
output_dir = str(tmp_path / "output")
- AutoRound(model=model_dir, scheme="W4A16", model_free=True).quantize_and_save(output_dir)
+ AutoRound(model=model_dir, scheme="W4A16", model_free=True, device_map="cpu").quantize_and_save(output_dir)
qc = _read_qconfig(output_dir)
assert qc["quant_method"] == "auto-round" and qc["bits"] == 4
@@ -1057,7 +1070,9 @@ class TestSchemeValidation:
def test_unsupported_raises(self, tmp_path, name):
model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, {"layer.weight": torch.randn(64, 128)})
with pytest.raises(ValueError):
- AutoRound(model=model_dir, model_free=True, scheme=name).quantize_and_save(str(tmp_path / "out"))
+ AutoRound(model=model_dir, model_free=True, scheme=name, device_map="cpu").quantize_and_save(
+ str(tmp_path / "out")
+ )
def test_is_model_free_supported_scheme(self):
for name in _SUPPORTED:
@@ -1088,6 +1103,8 @@ class TestCliAutoRouting:
[
"--model",
model_dir,
+ "--device",
+ "cpu",
"--scheme",
"W4A16",
"--iters",
@@ -1464,6 +1481,7 @@ class TestModelFreeAutoScheme:
model_free=True,
nsamples=1,
dataset=_make_local_calibration_dataset(tmp_path),
+ device_map="cpu",
)
ar.quantize_and_save(output_dir, format="auto_round")
@@ -1490,6 +1508,7 @@ class TestModelFreeAutoScheme:
model_free=True,
nsamples=1,
dataset=_make_local_calibration_dataset(tmp_path),
+ device_map="cpu",
)
ar.quantize_and_save(output_dir, format="llm_compressor")
@@ -1528,7 +1547,9 @@ class TestMXFPAutoRoundFormat:
}
model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
output_dir = str(tmp_path / "output")
- AutoRound(model=model_dir, scheme="MXFP4", model_free=True).quantize_and_save(output_dir, format="auto_round")
+ AutoRound(model=model_dir, scheme="MXFP4", model_free=True, device_map="cpu").quantize_and_save(
+ output_dir, format="auto_round"
+ )
qc = _read_qconfig(output_dir)
assert qc["quant_method"] == "auto-round"
assert qc["packing_format"] == "auto_round:llm_compressor"
@@ -1556,7 +1577,9 @@ class TestMXFPAutoRoundFormat:
}
model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
output_dir = str(tmp_path / "output")
- AutoRound(model=model_dir, scheme="MXFP8", model_free=True).quantize_and_save(output_dir, format="auto_round")
+ AutoRound(model=model_dir, scheme="MXFP8", model_free=True, device_map="cpu").quantize_and_save(
+ output_dir, format="auto_round"
+ )
qc = _read_qconfig(output_dir)
assert qc["quant_method"] == "auto-round"
assert qc["packing_format"] == "auto_round:llm_compressor"
@@ -1575,9 +1598,9 @@ class TestMXFPAutoRoundFormat:
}
model_dir = _make_model_dir(tmp_path, _LLAMA_CFG, tensors)
output_dir = str(tmp_path / "output")
- AutoRound(model=model_dir, scheme="MXFP4", model_free=True, quant_lm_head=True).quantize_and_save(
- output_dir, format="auto_round"
- )
+ AutoRound(
+ model=model_dir, scheme="MXFP4", model_free=True, quant_lm_head=True, device_map="cpu"
+ ).quantize_and_save(output_dir, format="auto_round")
qc = _read_qconfig(output_dir)
extra = qc.get("extra_config", {})
assert "lm_head" in extra
diff --git a/test/unit/test_cuda/advanced/test_evaluation.py b/test/unit/test_cuda/advanced/test_evaluation.py
index 31a87a84f..fe0b47894 100644
--- a/test/unit/test_cuda/advanced/test_evaluation.py
+++ b/test/unit/test_cuda/advanced/test_evaluation.py
@@ -48,7 +48,7 @@ class TestHFEvaluation:
f"--model {tiny_opt_model_path} --eval --tasks lambada_openai --limit 10",
)
- @pytest.mark.skip_ci(reason="The evaluation is time-consuming")
+ @pytest.mark.skip_ci(reason="Accuracy: The evaluation is time-consuming")
def test_iters_0_hf_backend(self, monkeypatch, tiny_opt_model_path):
"""Test quantization with iters=0 and HF backend evaluation."""
_run_auto_round_cli(
@@ -56,7 +56,7 @@ class TestHFEvaluation:
f"--model {tiny_opt_model_path} --iters 0 --disable_opt_rtn --tasks lambada_openai --limit 10",
)
- @pytest.mark.skip_ci(reason="The evaluation is time-consuming")
+ @pytest.mark.skip_ci(reason="Accuracy: The evaluation is time-consuming")
def test_iters_0_task_by_task(self, monkeypatch, tiny_opt_model_path):
"""Test quantization with iters=0 and task-by-task evaluation."""
_run_auto_round_cli(
diff --git a/test/unit/test_cuda/advanced/test_multiple_card.py b/test/unit/test_cuda/advanced/test_multiple_card.py
index c8ce4f7af..e84d95470 100644
--- a/test/unit/test_cuda/advanced/test_multiple_card.py
+++ b/test/unit/test_cuda/advanced/test_multiple_card.py
@@ -16,7 +16,7 @@ AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
[email protected]_ci(reason="multiple card test")
[email protected]_ci(reason="Resource: multiple card test")
class TestAutoRoundCli:
tasks = "lambada_openai"
@@ -61,7 +61,7 @@ class TestAutoRoundCli:
assert False, "cmd line test fail, please have a check"
[email protected]_ci(reason="multiple card test")
[email protected]_ci(reason="Resource: multiple card test")
class TestAutoRound:
@pytest.fixture(autouse=True)
diff --git a/test/unit/test_cuda/algorithms/test_alg_ext.py b/test/unit/test_cuda/algorithms/test_alg_ext.py
index 8b61e251a..3db2033fb 100644
--- a/test/unit/test_cuda/algorithms/test_alg_ext.py
+++ b/test/unit/test_cuda/algorithms/test_alg_ext.py
@@ -1,3 +1,4 @@
+import json
import shutil
import sys
from test.helpers import evaluate_accuracy, get_model_path
@@ -12,6 +13,15 @@ AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
+def _local_calibration_dataset(tmp_path):
+ dataset_path = tmp_path / "alg_ext_autoscheme_calibration.json"
+ dataset_path.write_text(
+ json.dumps(["Algorithm-extension local calibration text with enough tokens. " * 16]),
+ encoding="utf-8",
+ )
+ return str(dataset_path)
+
+
class TestAlgExt:
@pytest.fixture(autouse=True)
@@ -92,10 +102,13 @@ class TestAlgExt:
assert quantizer.enable_quanted_input is False
@pytest.mark.timeout(120)
- @pytest.mark.parametrize("scheme", ["MXFP4", "NVFP4", "W2A16G64", "gguf:q2_k_s,gguf:q4_k_s"])
- def test_all_support_dtype(self, scheme, tiny_qwen_model_path):
+ @pytest.mark.parametrize("scheme", ["MXFP4", "W2A16G64"])
+ def test_all_support_dtype(self, scheme, tiny_qwen_model_path, tmp_path, monkeypatch):
from auto_round.auto_scheme import AutoScheme
+ monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "1")
+ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "1")
+ calibration_dataset = _local_calibration_dataset(tmp_path)
avg_bits = 2 if scheme == "W2A16G64" else 4
scheme = AutoScheme(options=scheme, avg_bits=avg_bits, ignore_scale_zp_bits=True)
ar = AutoRound(
@@ -104,12 +117,13 @@ class TestAlgExt:
iters=1,
nsamples=1,
seqlen=32,
+ dataset=calibration_dataset,
enable_alg_ext=True,
enable_torch_compile=True,
)
ar.quantize()
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
+ @pytest.mark.skip_ci(reason="Architecture: Only tiny model is suggested")
@pytest.mark.skipif(reason="Time-consuming for accuracy evaluation")
def test_2bits(self):
model_name = get_model_path("facebook/opt-125m")
@@ -123,7 +137,7 @@ class TestAlgExt:
tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
evaluate_accuracy(model, tokenizer, threshold=0.22, batch_size=64)
- @pytest.mark.skip_ci(reason="Not necessary to test all case in CI")
+ @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all case in CI")
def test_cli(self, tiny_opt_model_path):
import os
diff --git a/test/unit/test_cuda/algorithms/test_auto_scheme.py b/test/unit/test_cuda/algorithms/test_auto_scheme.py
index 73945bd1e..194b5d12d 100644
--- a/test/unit/test_cuda/algorithms/test_auto_scheme.py
+++ b/test/unit/test_cuda/algorithms/test_auto_scheme.py
@@ -1,4 +1,5 @@
import copy
+import json
import re
import shutil
from test.helpers import evaluate_accuracy, get_model_path, get_tiny_model
@@ -15,6 +16,15 @@ from auto_round.utils import get_module
from ...envs import multi_card
+def _local_calibration_dataset(tmp_path):
+ dataset_path = tmp_path / "autoscheme_calibration.json"
+ dataset_path.write_text(
+ json.dumps(["AutoScheme local calibration text with enough tokens. " * 16]),
+ encoding="utf-8",
+ )
+ return str(dataset_path)
+
+
class TestAutoScheme:
@pytest.fixture(autouse=True)
@@ -29,40 +39,51 @@ class TestAutoScheme:
shutil.rmtree("runs", ignore_errors=True)
@pytest.mark.timeout(60)
- def test_gguf_k_0(self, tiny_qwen_model_path):
+ def test_gguf_k_0(self, tiny_qwen_model_path, tmp_path, monkeypatch):
+ monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "1")
+ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "1")
+ calibration_dataset = _local_calibration_dataset(tmp_path)
target_bits = 5.5
scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q4_K_M", "GGUF:Q8_0"))
- ar = AutoRound(model=tiny_qwen_model_path, scheme=scheme, iters=1, enable_alg_ext=True)
+ ar = AutoRound(
+ model=tiny_qwen_model_path,
+ scheme=scheme,
+ iters=1,
+ nsamples=1,
+ seqlen=8,
+ dataset=calibration_dataset,
+ enable_alg_ext=True,
+ )
ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")
- @pytest.mark.skip_ci(reason="not necessary to test all options")
+ @pytest.mark.skip_ci(reason="Matrix: not necessary to test all options")
def test_auto_scheme_export_gguf(self, tiny_qwen_model_path):
scheme = AutoScheme(avg_bits=3, options=("gguf:q2_k_s,gguf:q4_k_s"), nsamples=1, ignore_scale_zp_bits=True)
ar = AutoRound(model=tiny_qwen_model_path, scheme=scheme, iters=0, disable_opt_rtn=True, nsamples=1)
ar.quantize()
- @pytest.mark.skip_ci(reason="not necessary to test all options")
+ @pytest.mark.skip_ci(reason="Matrix: not necessary to test all options")
def test_gguf_k_1(self, tiny_qwen_model_path):
target_bits = 3.5
scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_1"))
ar = AutoRound(model=tiny_qwen_model_path, scheme=scheme, iters=1, enable_alg_ext=True)
ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")
- @pytest.mark.skip_ci(reason="not necessary to test all options")
+ @pytest.mark.skip_ci(reason="Matrix: not necessary to test all options")
def test_embedding_fallback(self, tiny_qwen_model_path):
target_bits = 5.0
scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q4_K_M", "GGUF:Q8_0"))
ar = AutoRound(model=tiny_qwen_model_path, scheme=scheme, iters=1, enable_alg_ext=True)
ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")
- @pytest.mark.skip_ci(reason="not necessary to test all options")
+ @pytest.mark.skip_ci(reason="Matrix: not necessary to test all options")
def test_gguf_export(self, tiny_qwen_model_path):
target_bits = 3
scheme = AutoScheme(avg_bits=target_bits, options=("GGUF:Q2_K_S", "GGUF:Q4_K_M"), ignore_scale_zp_bits=True)
ar = AutoRound(model=tiny_qwen_model_path, scheme=scheme, iters=0)
ar.quantize_and_save(self.save_dir, format="gguf:q2_k_s")
- @pytest.mark.skip_ci(reason="not necessary to test all options")
+ @pytest.mark.skip_ci(reason="Matrix: not necessary to test all options")
def test_gguf(self):
model_name = get_model_path("Qwen/Qwen3-8B")
model = get_tiny_model(model_name)
@@ -116,7 +137,7 @@ class TestAutoScheme:
print(avg_bits)
assert target_bits - 0.2 < avg_bits <= target_bits + 1e-3
- @pytest.mark.skip_ci(reason="multiple card test")
+ @pytest.mark.skip_ci(reason="Resource: multiple card test")
@multi_card
def test_multi_card(self, tiny_qwen_model_path):
target_bits = 4.5
@@ -135,7 +156,7 @@ class TestAutoScheme:
print(avg_bits)
assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3
- @pytest.mark.skip_ci(reason="multiple card test")
+ @pytest.mark.skip_ci(reason="Resource: multiple card test")
@multi_card
def test_multi_card_1(self, tiny_qwen_model_path):
target_bits = 4.5
@@ -159,7 +180,7 @@ class TestAutoScheme:
print(avg_bits)
assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3
- @pytest.mark.skip_ci(reason="multiple card test")
+ @pytest.mark.skip_ci(reason="Resource: multiple card test")
@multi_card
def test_dict_device_map(self, tiny_qwen_model_path):
target_bits = 8.25
@@ -174,7 +195,7 @@ class TestAutoScheme:
print(avg_bits)
assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3
- @pytest.mark.skip_ci(reason="Not necessary to test the corner case in CI")
+ @pytest.mark.skip_ci(reason="Matrix: Not necessary to test the corner case in CI")
def test_min_target_bits(self, tiny_opt_model_path):
target_bits = 4.644
scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16"))
@@ -184,7 +205,7 @@ class TestAutoScheme:
print(avg_bits)
assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
+ @pytest.mark.skip_ci(reason="Architecture: Only tiny model is suggested")
def test_max_target_bits(self):
target_bits = 8.025
model_path = get_model_path("facebook/opt-125m")
@@ -195,7 +216,7 @@ class TestAutoScheme:
print(avg_bits)
assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3
- @pytest.mark.skip_ci(reason="Not necessary to test the corner case in CI")
+ @pytest.mark.skip_ci(reason="Matrix: Not necessary to test the corner case in CI")
def test_patch_scheme(self, tiny_opt_model_path):
target_bits = 5
scheme = AutoScheme(avg_bits=target_bits, options=("MXFP4", "W8A16"))
@@ -210,7 +231,7 @@ class TestAutoScheme:
print(avg_bits)
assert target_bits - 0.1 < avg_bits <= target_bits + 1e-3
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
+ @pytest.mark.skip_ci(reason="Architecture: Only tiny model is suggested")
def test_layer_config(self):
target_bits = 3.0
model_name = get_model_path("facebook/opt-125m")
@@ -298,7 +319,7 @@ class TestAutoScheme:
has_nonzero = any(opt[2] > 0 for opt in head_scores)
assert has_nonzero, f"lm_head mix_score is 0 for all schemes: {head_scores}"
- @pytest.mark.skip_ci(reason="The evaluation is time-consuming")
+ @pytest.mark.skip_ci(reason="Accuracy: The evaluation is time-consuming")
def test_auto_scheme_export(self):
model_name = get_model_path("facebook/opt-125m")
scheme = AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "W8A16", "BF16"))
@@ -306,7 +327,7 @@ class TestAutoScheme:
_, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir)
evaluate_accuracy(quantized_model_path, threshold=0.25)
- @pytest.mark.skip_ci(reason="The evaluation is time-consuming")
+ @pytest.mark.skip_ci(reason="Accuracy: The evaluation is time-consuming")
def test_enable_torch_compile(self):
model_name = get_model_path("facebook/opt-125m")
scheme = AutoScheme(avg_bits=2, options=("W2A16"), ignore_scale_zp_bits=True)
@@ -314,7 +335,7 @@ class TestAutoScheme:
_, quantized_model_path = ar.quantize_and_save(output_dir=self.save_dir)
evaluate_accuracy(quantized_model_path, threshold=0.10)
- @pytest.mark.skip_ci(reason="Time-consuming: two quantizations plus two lm_eval runs; covered by nightly")
+ @pytest.mark.skip_ci(reason="Accuracy: Time-consuming: two quantizations plus two lm_eval runs; covered by nightly")
def test_mixed_bits_get_scoring(self):
"""Verify that AutoScheme scoring produces accuracy above a known reference threshold for mixed-bit
quantization.
diff --git a/test/unit/test_cuda/algorithms/test_awq.py b/test/unit/test_cuda/algorithms/test_awq.py
index 524501623..b207bb2a4 100644
--- a/test/unit/test_cuda/algorithms/test_awq.py
+++ b/test/unit/test_cuda/algorithms/test_awq.py
@@ -51,7 +51,7 @@ class TestAWQNormalLLM:
assert cfg.awq_seqlen == 128
- @pytest.mark.skip_ci(reason="CPU-only AWQ algorithm smoke; run in common/nightly coverage")
+ @pytest.mark.skip_ci(reason="Coverage: CPU-only AWQ algorithm smoke; run in common/nightly coverage")
@pytest.mark.timeout(90)
def test_awq_w4a16_quantize_and_inference(self, tiny_opt_model_path):
"""W4A16 AWQ quantization produces valid layer_config and the model can generate.
@@ -81,7 +81,7 @@ class TestAWQNormalLLM:
output = generate_prompt(model, tokenizer, device=device)
assert len(output) > 0, "Model should produce non-empty output"
- @pytest.mark.skip_ci(reason="CPU-only AWQ export smoke; run in common/nightly coverage")
+ @pytest.mark.skip_ci(reason="Coverage: CPU-only AWQ export smoke; run in common/nightly coverage")
def test_awq_w4a16_export_default_scheme(self, tiny_opt_model_path):
"""Default W4A16 scheme export: quantization_config has bits=4, group_size=128."""
ar = AutoRound(
@@ -103,7 +103,7 @@ class TestAWQNormalLLM:
assert qconfig["group_size"] == 128
assert "auto-round" in qconfig["quant_method"]
- @pytest.mark.skip_ci(reason="CPU-only AWQ export smoke; run in common/nightly coverage")
+ @pytest.mark.skip_ci(reason="Coverage: CPU-only AWQ export smoke; run in common/nightly coverage")
def test_awq_w4a16_export_args_check(self, tiny_opt_model_path):
"""Saved quantization_config (bits/group_size/sym/quant_method) matches the input parameters.
@@ -164,7 +164,7 @@ class TestAWQNonIntegerSchemes:
MXFP/NVFP scheme.
"""
- @pytest.mark.skip_ci(reason="CPU-only AWQ datatype smoke; run in common/nightly coverage")
+ @pytest.mark.skip_ci(reason="Coverage: CPU-only AWQ datatype smoke; run in common/nightly coverage")
@pytest.mark.timeout(60)
@pytest.mark.parametrize("scheme", ["MXFP4", "NVFP4"])
def test_awq_non_integer_scheme_smoke(self, tiny_opt_model_path, scheme):
@@ -204,7 +204,7 @@ class TestAWQW8A8LLMCompressor:
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
- @pytest.mark.skip_ci(reason="LLM-Compressor export is covered by integration CI")
+ @pytest.mark.skip_ci(reason="Coverage: LLM-Compressor export is covered by integration CI")
def test_awq_w8a8_llmc_export(self, tiny_opt_model_path):
"""W8A8 AWQ -> llm_compressor: verify compressed-tensors metadata and int8 saved weights.
@@ -256,7 +256,7 @@ class TestAWQMoE:
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
- def test_awq_moe_dynamic_smoothing(self, tiny_qwen_moe_model_path):
+ def test_awq_moe_dynamic_smoothing(self, micro_qwen_moe_model_path):
"""AWQ dynamic smoothing should resolve mappings on a MoE model without error.
Pure mapping-resolution logic, no inference -- runs once on cpu.
@@ -265,7 +265,7 @@ class TestAWQMoE:
from auto_round.algorithms.transforms.awq.mappings import resolve_mappings
model = AutoModelForCausalLM.from_pretrained(
- tiny_qwen_moe_model_path,
+ micro_qwen_moe_model_path,
torch_dtype=torch.bfloat16,
device_map=device,
trust_remote_code=True,
@@ -741,7 +741,7 @@ class TestAWQMoE:
@requires_cuda
@pytest.mark.timeout(420)
- def test_awq_moe_quantized_layers_check(self, tiny_qwen_moe_model_path):
+ def test_awq_moe_quantized_layers_check(self, micro_qwen_moe_model_path):
"""AWQ on MoE: expert layers should be quantized, gates/routers stay FP.
Algorithm/config correctness, but MoE quantization is slow enough (real tuning over
@@ -750,7 +750,7 @@ class TestAWQMoE:
"""
device = "cuda"
ar = AutoRound(
- tiny_qwen_moe_model_path,
+ micro_qwen_moe_model_path,
scheme="W4A16",
alg_configs=AWQConfig(n_grid=1),
n_grid=1,
@@ -785,7 +785,7 @@ class TestAWQMoE:
# TODO: Investigate and fix the excessive test runtime instead of relying on an increased timeout.
@requires_cuda
@pytest.mark.timeout(400)
- def test_awq_moe_save_quant_config(self, tiny_qwen_moe_model_path):
+ def test_awq_moe_save_quant_config(self, micro_qwen_moe_model_path):
"""AWQ MoE: saved quantization_config should be consistent and loadable.
Same rationale as test_awq_moe_quantized_layers_check: config correctness, but MoE
@@ -793,7 +793,7 @@ class TestAWQMoE:
"""
device = "cuda"
ar = AutoRound(
- tiny_qwen_moe_model_path,
+ micro_qwen_moe_model_path,
scheme="W4A16",
alg_configs=AWQConfig(n_grid=1),
n_grid=1,
@@ -903,7 +903,7 @@ class TestAWQWeightClip:
output = generate_prompt(model, tokenizer, device=device)
assert len(output) > 0, "Clipped model should produce non-empty output"
- @pytest.mark.skip_ci(reason="CPU-only AWQ clip/SignRound smoke; run in common/nightly coverage")
+ @pytest.mark.skip_ci(reason="Coverage: CPU-only AWQ clip/SignRound smoke; run in common/nightly coverage")
def test_awq_clip_as_init_signround(self, tiny_opt_model_path):
"""clip_as_init: clip is kept on the model context and initializes SignRound's range.
diff --git a/test/unit/test_cuda/algorithms/test_awq_accuracy.py b/test/unit/test_cuda/algorithms/test_awq_accuracy.py
index 86b81801c..efd3619c7 100644
--- a/test/unit/test_cuda/algorithms/test_awq_accuracy.py
+++ b/test/unit/test_cuda/algorithms/test_awq_accuracy.py
@@ -40,7 +40,7 @@ class TestAWQAccuracy:
def teardown_class(cls):
shutil.rmtree("runs", ignore_errors=True)
- @pytest.mark.skip_ci(reason="Time-consuming lm_eval accuracy check; covered by nightly")
+ @pytest.mark.skip_ci(reason="Accuracy: Time-consuming lm_eval accuracy check; covered by nightly")
def test_awq_w4a16_lmeval(self):
"""AWQ W4A16 on OPT-125m: lambada_openai accuracy check."""
ar = AutoRound(
diff --git a/test/unit/test_cuda/algorithms/test_block_runner.py b/test/unit/test_cuda/algorithms/test_block_runner.py
index f96a3e1a9..73f5eebca 100644
--- a/test/unit/test_cuda/algorithms/test_block_runner.py
+++ b/test/unit/test_cuda/algorithms/test_block_runner.py
@@ -16,7 +16,7 @@ import shutil
import pytest
import torch
-from transformers import AutoModelForCausalLM
+from transformers import AutoModelForCausalLM, AutoRoundConfig
from auto_round import AutoRound
@@ -31,6 +31,20 @@ def _has_packed_weight(model):
return False
+def _load_with_torch_backend(quantized_model_path):
+ """Reload generic export coverage with the reference backend.
+
+ Optimized backend selection and kernel compilation are covered in their
+ dedicated CUDA backend tests.
+ """
+ return AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ device_map="cuda:0",
+ trust_remote_code=True,
+ quantization_config=AutoRoundConfig(backend="torch"),
+ )
+
+
class TestBlockRunnerGpu:
@pytest.fixture(autouse=True)
def _save_dir(self, tmp_path):
@@ -39,7 +53,7 @@ class TestBlockRunnerGpu:
shutil.rmtree(self.save_dir, ignore_errors=True)
@pytest.mark.timeout(180)
- def test_3bit_asym_sign_sgd_reload(self, tiny_opt_model_path):
+ def test_3bit_asym_sign_sgd_reload(self, tiny_opt_model_path, dataloader):
autoround = AutoRound(
tiny_opt_model_path,
bits=3,
@@ -47,14 +61,22 @@ class TestBlockRunnerGpu:
sym=False,
iters=2,
nsamples=2,
- seqlen=16,
+ seqlen=8,
+ dataset=dataloader,
)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = _load_with_torch_backend(quantized_model_path)
assert isinstance(model, torch.nn.Module)
assert _has_packed_weight(model)
+ input_ids = torch.randint(0, 1000, (1, 16), device="cuda:0")
+ with torch.no_grad():
+ out = model(input_ids)
+ assert out.logits.shape[0] == 1
+ @pytest.mark.skip_ci(
+ reason="Matrix: group_size=64 packing is covered by test_autoround_int_export; retain full Sign-SGD case weekly"
+ )
@pytest.mark.timeout(180)
def test_group64_sign_sgd_reload(self, tiny_opt_model_path):
autoround = AutoRound(
@@ -68,10 +90,13 @@ class TestBlockRunnerGpu:
)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = _load_with_torch_backend(quantized_model_path)
assert isinstance(model, torch.nn.Module)
assert _has_packed_weight(model)
+ @pytest.mark.skip_ci(
+ reason="Coverage: symmetric INT export/reload is covered by test_autoround_int_export; retain tuning case weekly"
+ )
@pytest.mark.timeout(180)
def test_4bit_sym_group128_reload(self, tiny_opt_model_path):
autoround = AutoRound(
@@ -85,10 +110,13 @@ class TestBlockRunnerGpu:
)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = _load_with_torch_backend(quantized_model_path)
assert isinstance(model, torch.nn.Module)
assert _has_packed_weight(model)
+ @pytest.mark.skip_ci(
+ reason="Coverage: asymmetric INT2 runtime is covered by the Triton backend smoke; retain full tuning case weekly"
+ )
@pytest.mark.timeout(180)
def test_2bit_asym_group32_reload(self, tiny_opt_model_path):
autoround = AutoRound(
@@ -102,7 +130,7 @@ class TestBlockRunnerGpu:
)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = _load_with_torch_backend(quantized_model_path)
assert isinstance(model, torch.nn.Module)
assert _has_packed_weight(model)
input_ids = torch.randint(0, 1000, (1, 16), device="cuda:0")
@@ -110,6 +138,9 @@ class TestBlockRunnerGpu:
out = model(input_ids)
assert out.logits.shape[0] == 1
+ @pytest.mark.skip_ci(
+ reason="Coverage: test_3bit_asym_sign_sgd_reload already verifies tuned reload and forward; retain longer convergence case weekly"
+ )
@pytest.mark.timeout(180)
def test_sign_sgd_forward_after_reload(self, tiny_opt_model_path):
"""Sign-SGD quantized model must forward-pass on GPU after reload."""
@@ -124,7 +155,7 @@ class TestBlockRunnerGpu:
)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = _load_with_torch_backend(quantized_model_path)
input_ids = torch.randint(0, 1000, (1, 16), device="cuda:0")
with torch.no_grad():
out = model(input_ids)
diff --git a/test/unit/test_cuda/algorithms/test_config_resolver.py b/test/unit/test_cuda/algorithms/test_config_resolver.py
index 0469f15d7..0d55a5a94 100644
--- a/test/unit/test_cuda/algorithms/test_config_resolver.py
+++ b/test/unit/test_cuda/algorithms/test_config_resolver.py
@@ -16,7 +16,7 @@ import shutil
import pytest
import torch
-from transformers import AutoModelForCausalLM
+from transformers import AutoConfig, AutoModelForCausalLM
from auto_round import AutoRound
@@ -36,8 +36,8 @@ class TestConfigResolverGpu:
autoround = AutoRound(tiny_opt_model_path, scheme=scheme, iters=0, disable_opt_rtn=True, nsamples=1, seqlen=16)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
- assert getattr(model.config.quantization_config, "quant_method", None) == "auto-round"
+ config = AutoConfig.from_pretrained(quantized_model_path, trust_remote_code=True)
+ assert getattr(config, "quantization_config", {}).get("quant_method") == "auto-round"
@pytest.mark.timeout(180)
def test_layer_config_override(self, tiny_opt_model_path):
diff --git a/test/unit/test_cuda/auto_scheme/test_auto_scheme.py b/test/unit/test_cuda/auto_scheme/test_auto_scheme.py
index bbcedfec7..e6f8d1c3e 100644
--- a/test/unit/test_cuda/auto_scheme/test_auto_scheme.py
+++ b/test/unit/test_cuda/auto_scheme/test_auto_scheme.py
@@ -12,17 +12,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import json
import shutil
import pytest
import torch
-from transformers import AutoModelForCausalLM
+from transformers import AutoModelForCausalLM, AutoRoundConfig
from auto_round import AutoRound, AutoScheme
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires CUDA")
+def _local_calibration_dataset(tmp_path):
+ dataset_path = tmp_path / "autoscheme_calibration.json"
+ dataset_path.write_text(
+ json.dumps(
+ [
+ "The quick brown fox jumps over the lazy dog. " * 32,
+ "Quantization calibration needs distinct activation patterns. " * 32,
+ "AutoRound assigns mixed precision to transformer layers. " * 32,
+ "A compact local dataset keeps CUDA unit tests deterministic. " * 32,
+ ]
+ ),
+ encoding="utf-8",
+ )
+ return str(dataset_path)
+
+
class TestAutoSchemeGpu:
@pytest.fixture(autouse=True)
def _save_dir(self, tmp_path):
@@ -31,43 +48,63 @@ class TestAutoSchemeGpu:
shutil.rmtree(self.save_dir, ignore_errors=True)
@pytest.mark.timeout(240)
- def test_auto_scheme_quantize_reload(self, tiny_qwen_model_path):
- scheme = AutoScheme(avg_bits=5.0, options=("W4A16", "W8A16"))
+ def test_auto_scheme_quantize_reload(self, tiny_qwen_model_path, tmp_path, monkeypatch):
+ monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "1")
+ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "1")
+ calibration_dataset = _local_calibration_dataset(tmp_path)
+ target = 5.0
+ scheme = AutoScheme(avg_bits=target, options=("W4A16", "W8A16"))
autoround = AutoRound(
tiny_qwen_model_path,
scheme=scheme,
iters=0,
disable_opt_rtn=True,
+ disable_model_free=True,
nsamples=1,
seqlen=16,
+ dataset=calibration_dataset,
+ )
+ quantized_model, quantized_model_path = autoround.quantize_and_save(
+ output_dir=self.save_dir, format="auto_round"
)
- _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ device_map="cuda:0",
+ trust_remote_code=True,
+ quantization_config=AutoRoundConfig(backend="torch"),
+ )
assert isinstance(model, torch.nn.Module)
- @pytest.mark.timeout(240)
- def test_auto_scheme_avg_bits_in_range(self, tiny_qwen_model_path):
- """The auto-generated mixed scheme must land near the requested avg_bits."""
+ @pytest.mark.timeout(120)
+ def test_auto_scheme_avg_bits_in_range(self, tiny_qwen_model_path, tmp_path, monkeypatch):
+ """The generated mixed scheme lands near the requested average bit-width."""
from auto_round.auto_scheme.utils import compute_avg_bits_for_model
+ monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "4")
+ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "1")
+ calibration_dataset = _local_calibration_dataset(tmp_path)
target = 5.0
- scheme = AutoScheme(avg_bits=target, options=("W4A16", "W8A16"))
autoround = AutoRound(
tiny_qwen_model_path,
- scheme=scheme,
+ scheme=AutoScheme(avg_bits=target, options=("W4A16", "W8A16")),
iters=0,
disable_opt_rtn=True,
+ disable_model_free=True,
nsamples=1,
seqlen=16,
+ dataset=calibration_dataset,
)
model, _ = autoround.quantize()
avg_bits, _ = compute_avg_bits_for_model(model, ignore_scale_zp_bits=True)
assert target - 0.3 < avg_bits <= target + 0.3
@pytest.mark.timeout(240)
- def test_auto_scheme_iters1_reload(self, tiny_qwen_model_path):
+ def test_auto_scheme_iters1_reload(self, tiny_qwen_model_path, tmp_path, monkeypatch):
"""Auto scheme with sign-SGD (iters>0) + reload."""
+ monkeypatch.setenv("AR_AUTO_SCHEME_NSAMPLES", "1")
+ monkeypatch.setenv("AR_AUTO_SCHEME_BATCH_SIZE", "1")
+ calibration_dataset = _local_calibration_dataset(tmp_path)
scheme = AutoScheme(avg_bits=5.0, options=("W4A16", "W8A16"))
autoround = AutoRound(
tiny_qwen_model_path,
@@ -75,8 +112,14 @@ class TestAutoSchemeGpu:
iters=1,
nsamples=1,
seqlen=16,
+ dataset=calibration_dataset,
)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ device_map="cuda:0",
+ trust_remote_code=True,
+ quantization_config=AutoRoundConfig(backend="torch"),
+ )
assert isinstance(model, torch.nn.Module)
diff --git a/test/unit/test_cuda/backends/test_exllamav2_backend.py b/test/unit/test_cuda/backends/test_exllamav2_backend.py
index 85d9314f7..434e7db08 100644
--- a/test/unit/test_cuda/backends/test_exllamav2_backend.py
+++ b/test/unit/test_cuda/backends/test_exllamav2_backend.py
@@ -47,26 +47,19 @@ class TestAutoRoundexllamaBackend:
)
quantization_config = AutoRoundConfig(backend="gptqmodel:exllamav2")
- model = AutoModelForCausalLM.from_pretrained(
- quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config
- )
-
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- model_infer(model, tokenizer)
- evaluate_accuracy(model, tokenizer, threshold=0.35, batch_size=16)
- torch.cuda.empty_cache()
-
- model = AutoModelForCausalLM.from_pretrained(
- quantized_model_path, torch_dtype=torch.bfloat16, device_map="auto", quantization_config=quantization_config
- )
-
- tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
- model_infer(model, tokenizer)
- evaluate_accuracy(model, tokenizer, threshold=0.35, batch_size=16)
- torch.cuda.empty_cache()
-
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ # Keep the full accuracy gate for the primary dtype. BF16 uses the same
+ # exported weights and backend; a bounded accuracy smoke detects dtype-specific
+ # loading or kernel regressions without duplicating the full evaluation.
+ for dtype, limit in ((torch.float16, None), (torch.bfloat16, 10)):
+ model = AutoModelForCausalLM.from_pretrained(
+ quantized_model_path, torch_dtype=dtype, device_map="auto", quantization_config=quantization_config
+ )
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ model_infer(model, tokenizer)
+ evaluate_accuracy(model, tokenizer, threshold=0.35, batch_size=16, limit=limit)
+ torch.cuda.empty_cache()
+
+ @pytest.mark.skip_ci(reason="Accuracy: Only tiny model is suggested; Time-consuming; Accuracy evaluation")
@require_autogptq
@require_package_version_ut("torch", "<2.6.0")
def test_gptq_exllamav2_4bits_sym(self, dataloader):
@@ -96,8 +89,7 @@ class TestAutoRoundexllamaBackend:
evaluate_accuracy(model, tokenizer, threshold=0.27, batch_size=16)
torch.cuda.empty_cache()
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Only tiny model is suggested; Time-consuming; Accuracy evaluation")
@require_autogptq
@require_package_version_ut("torch", "<2.6.0")
@pytest.mark.parametrize("group_size", [-1, 32, 64, 128, 256, 1024])
@@ -128,7 +120,9 @@ class TestAutoRoundexllamaBackend:
torch.cuda.empty_cache()
@require_gptqmodel
- @pytest.mark.skip_ci(reason="AWQ ExLlamaV2 matrix is covered in nightly; keep the native asym smoke in PR CI")
+ @pytest.mark.skip_ci(
+ reason="Backend/JIT: AWQ ExLlamaV2 matrix is covered in nightly; keep the native asym smoke in PR CI"
+ )
@pytest.mark.timeout(90)
def test_gptqmodel_awq_exllamav2_4bits_asym(self, dataloader):
"""Test AWQ quantization with gptqmodel:awq_exllamav2 backend (bfloat16 inference)."""
@@ -161,8 +155,7 @@ class TestAutoRoundexllamaBackend:
torch.cuda.empty_cache()
@require_gptqmodel
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Only tiny model is suggested; Time-consuming; Accuracy evaluation")
def test_gptqmodel_awq_exllamav2_4bits_sym(self, dataloader):
"""Test AWQ quantization with gptqmodel:awq_exllamav2 backend (bfloat16 inference, symmetric)."""
model_path = get_model_path("facebook/opt-125m")
diff --git a/test/unit/test_cuda/backends/test_gptqmodel_awq_backend.py b/test/unit/test_cuda/backends/test_gptqmodel_awq_backend.py
index a72feaad3..259521c2b 100644
--- a/test/unit/test_cuda/backends/test_gptqmodel_awq_backend.py
+++ b/test/unit/test_cuda/backends/test_gptqmodel_awq_backend.py
@@ -34,7 +34,7 @@ class TestGptqmodelAwqTorchBackend:
shutil.rmtree(self.save_dir, ignore_errors=True)
@require_gptqmodel
- @pytest.mark.skip_ci(reason="Not necessary to test low priority backend in CI")
+ @pytest.mark.skip_ci(reason="Backend/JIT: Not necessary to test low priority backend in CI")
def test_gptqmodel_awq_torch_4bits_group_size_16(self, dataloader):
"""AWQ-quantized (group_size=16) model loads via the gptqmodel:awq_torch backend and meets accuracy."""
model_path = get_model_path("facebook/opt-125m")
diff --git a/test/unit/test_cuda/backends/test_marlin_backend.py b/test/unit/test_cuda/backends/test_marlin_backend.py
index 1ba9f509f..44da32583 100644
--- a/test/unit/test_cuda/backends/test_marlin_backend.py
+++ b/test/unit/test_cuda/backends/test_marlin_backend.py
@@ -62,8 +62,7 @@ class TestAutoRoundMarlinBackend:
evaluate_accuracy(model, tokenizer, threshold=0.27, batch_size=16)
torch.cuda.empty_cache()
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Only tiny model is suggested; Time-consuming; Accuracy evaluation")
def test_marlin_group_size(self, dataloader):
for group_size in [-1, 64]:
print(f"{group_size}!!!!!!!!!!!!!!!!!")
@@ -173,7 +172,9 @@ class TestAutoRoundMarlinBackend:
# shutil.rmtree("./saved", ignore_errors=True)
@require_gptqmodel
- @pytest.mark.skip_ci(reason="AWQ Marlin is a format/backend matrix case; standard Marlin smoke remains in PR CI")
+ @pytest.mark.skip_ci(
+ reason="Backend/JIT: AWQ Marlin is a format/backend matrix case; standard Marlin smoke remains in PR CI"
+ )
@pytest.mark.timeout(90)
def test_gptqmodel_awq_marlin_4bits_sym(self):
"""Test AWQ quantization with gptqmodel:awq_marlin backend (sym-only, float16)."""
@@ -205,8 +206,7 @@ class TestAutoRoundMarlinBackend:
torch.cuda.empty_cache()
@require_gptqmodel
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Only tiny model is suggested; Time-consuming; Accuracy evaluation")
@pytest.mark.parametrize("group_size", [32, 64, 128])
def test_gptqmodel_awq_marlin_group_size(self, group_size):
"""Test AWQ marlin backend with different group sizes."""
diff --git a/test/unit/test_cuda/backends/test_torch_backend.py b/test/unit/test_cuda/backends/test_torch_backend.py
index 5ec8081f6..e9b689692 100644
--- a/test/unit/test_cuda/backends/test_torch_backend.py
+++ b/test/unit/test_cuda/backends/test_torch_backend.py
@@ -80,7 +80,9 @@ class TestTorchBackendFunctional:
tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
model_infer(model, tokenizer)
- @pytest.mark.skip_ci(reason="Not necessary to test both symmetric and asymmetric for 3-bit quantization in CI")
+ @pytest.mark.skip_ci(
+ reason="Matrix: Not necessary to test both symmetric and asymmetric for 3-bit quantization in CI"
+ )
@pytest.mark.parametrize("device", _AVAILABLE_DEVICES)
def test_torch_backend_3bit_sym(self, tiny_opt_model_path, dataloader, device):
"""3-bit symmetric model loads via the torch backend and generates text."""
diff --git a/test/unit/test_cuda/backends/test_torch_backend_accuracy.py b/test/unit/test_cuda/backends/test_torch_backend_accuracy.py
index 0da8587dc..82cf60b72 100644
--- a/test/unit/test_cuda/backends/test_torch_backend_accuracy.py
+++ b/test/unit/test_cuda/backends/test_torch_backend_accuracy.py
@@ -101,7 +101,7 @@ class TestTorchBackendAccuracy:
evaluate_accuracy(model, tokenizer, threshold=0.35, batch_size=16)
torch.cuda.empty_cache()
- @pytest.mark.skip_ci(reason="Time-consuming; covered by the RTN accuracy check above")
+ @pytest.mark.skip_ci(reason="Accuracy: Time-consuming; covered by the RTN accuracy check above")
def test_torch_backend_accuracy_sym_tuning(self, dataloader):
"""Tuned (iters=1) 4-bit sym model meets the accuracy threshold."""
model = AutoModelForCausalLM.from_pretrained(self.model_name, dtype="auto", trust_remote_code=True)
diff --git a/test/unit/test_cuda/backends/test_triton_backend.py b/test/unit/test_cuda/backends/test_triton_backend.py
index 95d57937c..6d689db5c 100644
--- a/test/unit/test_cuda/backends/test_triton_backend.py
+++ b/test/unit/test_cuda/backends/test_triton_backend.py
@@ -1,5 +1,5 @@
import shutil
-from test.helpers import evaluate_accuracy, get_model_path, model_infer
+from test.helpers import evaluate_accuracy, model_infer
import pytest
import torch
@@ -11,10 +11,6 @@ from ...envs import require_greater_than_050
class TestAutoRoundTritonBackend:
- @classmethod
- def setup_class(self):
- self.model_name = get_model_path("facebook/opt-125m")
-
@pytest.fixture(autouse=True)
def _save_dir(self, tmp_path):
self.save_folder = str(tmp_path / "saved")
@@ -25,31 +21,39 @@ class TestAutoRoundTritonBackend:
def teardown_class(self):
shutil.rmtree("runs", ignore_errors=True)
- # Keep 2 bits symmetric test for triton backend since it's a special configuration and we want to make sure it's working well.
- # @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- # @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
@require_greater_than_050
- @pytest.mark.timeout(300)
- def test_tritonv2_2bits_asym(self):
- model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(self.model_name, trust_remote_code=True)
- bits, group_size, sym = 2, 32, False
- autoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)
- quantized_model_path = self.save_folder
- _, quantized_model_path = autoround.quantize_and_save(output_dir=quantized_model_path)
+ @pytest.mark.timeout(120)
+ def test_tritonv2_2bits_asym(self, tiny_opt_model_path):
+ """A tuned INT2 asymmetric export reloads and generates with Triton.
+
+ This is the PR smoke for the Triton kernel. Accuracy evaluation and the
+ full OPT-125M model belong to the weekly backend matrix below; neither
+ changes Triton export, load, or generation behavior.
+ """
+ model = AutoModelForCausalLM.from_pretrained(tiny_opt_model_path, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path, trust_remote_code=True)
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=2,
+ group_size=32,
+ sym=False,
+ iters=1,
+ nsamples=1,
+ seqlen=8,
+ dataset=["local Triton calibration sample with enough tokens for quantization"],
+ )
+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_folder)
quantization_config = AutoRoundConfig(backend="tritonv2")
model = AutoModelForCausalLM.from_pretrained(
quantized_model_path, torch_dtype=torch.float16, device_map="auto", quantization_config=quantization_config
)
-
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
model_infer(model, tokenizer)
- evaluate_accuracy(model, tokenizer, threshold=0.19, batch_size=16)
torch.cuda.empty_cache()
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Only tiny model is suggested; Time-consuming; Accuracy evaluation")
@require_greater_than_050
def test_tritonv2_4bits_asym(self, dataloader):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
@@ -80,8 +84,7 @@ class TestAutoRoundTritonBackend:
evaluate_accuracy(model, tokenizer, threshold=0.34, batch_size=16)
torch.cuda.empty_cache()
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Only tiny model is suggested; Time-consuming; Accuracy evaluation")
@require_greater_than_050
def test_tritonv2_4bits_sym(self, dataloader):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
@@ -119,8 +122,7 @@ class TestAutoRoundTritonBackend:
evaluate_accuracy(model, tokenizer, threshold=0.26, batch_size=16)
torch.cuda.empty_cache()
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Only tiny model is suggested; Time-consuming; Accuracy evaluation")
@require_greater_than_050
def test_tritonv2_8bits_sym(self):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
@@ -149,8 +151,7 @@ class TestAutoRoundTritonBackend:
evaluate_accuracy(model, tokenizer, threshold=0.27, batch_size=16)
torch.cuda.empty_cache()
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Only tiny model is suggested; Time-consuming; Accuracy evaluation")
@require_greater_than_050
def test_tritonv2_2bits_sym(self):
model = AutoModelForCausalLM.from_pretrained(self.model_name, torch_dtype="auto", trust_remote_code=True)
diff --git a/test/unit/test_cuda/calibration/test_calib_dataset.py b/test/unit/test_cuda/calibration/test_calib_dataset.py
index fd90cdfec..f3b665031 100644
--- a/test/unit/test_cuda/calibration/test_calib_dataset.py
+++ b/test/unit/test_cuda/calibration/test_calib_dataset.py
@@ -1,20 +1,22 @@
-import json
-import os
-import shutil
-
import pytest
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoTokenizer
-from auto_round import AutoRound
+from auto_round.calib_dataset import get_dataloader
class TestLocalCalibDataset:
@pytest.mark.timeout(150)
- def test_combine_dataset(self, tiny_opt_model_path):
- dataset = "NeelNanda/pile-10k" + ",BAAI/CCI3-HQ" + ",madao33/new-title-chinese"
- bits, group_size, sym = 4, 128, True
- autoround = AutoRound(
- tiny_opt_model_path, bits=bits, group_size=group_size, sym=sym, iters=2, seqlen=128, dataset=dataset
- )
- autoround.quantize()
+ def test_combine_dataset(self, tiny_opt_model_path, monkeypatch):
+ # Subprocess lifecycle is covered by test_calib_dataset_subprocess.py. The
+ # streaming sources below cannot reuse its cache, so avoid loading each twice.
+ monkeypatch.setenv("AR_DISABLE_DATASET_SUBPROCESS", "1")
+ dataset = "NeelNanda/pile-10k:num=1,BAAI/CCI3-HQ:num=1,madao33/new-title-chinese:num=1"
+ tokenizer = AutoTokenizer.from_pretrained(tiny_opt_model_path)
+
+ dataloader = get_dataloader(tokenizer, seqlen=128, dataset_name=dataset, bs=3, nsamples=3)
+ batch = next(iter(dataloader))
+
+ assert len(dataloader.dataset) == 3
+ assert batch is not None
+ assert batch["input_ids"].shape == (3, 128)
+ assert batch["attention_mask"].shape == (3, 128)
diff --git a/test/unit/test_cuda/calibration/test_customized_data.py b/test/unit/test_cuda/calibration/test_customized_data.py
index cf2ee792e..6bf0acfc6 100644
--- a/test/unit/test_cuda/calibration/test_customized_data.py
+++ b/test/unit/test_cuda/calibration/test_customized_data.py
@@ -23,7 +23,7 @@ class TestCustomizedData:
yield
shutil.rmtree("runs", ignore_errors=True)
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
+ @pytest.mark.skip_ci(reason="Architecture: Only tiny model is suggested")
def test_mixed_attention_mask(self):
model_name = get_model_path("Qwen/Qwen3-0.6B")
tokenizer = AutoTokenizer.from_pretrained(model_name)
@@ -40,7 +40,7 @@ class TestCustomizedData:
ar = AutoRound(model_name, dataset=inputs, seqlen=9)
ar.quantize()
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
+ @pytest.mark.skip_ci(reason="Architecture: Only tiny model is suggested")
def test_batch_encoding(self):
model_name = get_model_path("Qwen/Qwen3-0.6B")
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype="auto")
diff --git a/test/unit/test_cuda/export/test_auto_awq_format.py b/test/unit/test_cuda/export/test_auto_awq_format.py
index b805fadd9..20a2639f5 100644
--- a/test/unit/test_cuda/export/test_auto_awq_format.py
+++ b/test/unit/test_cuda/export/test_auto_awq_format.py
@@ -53,7 +53,7 @@ class TestAutoRound:
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")
assert model is not None, "Loaded model should not be None."
- @pytest.mark.skip_ci(reason="Only tiny model is suggested")
+ @pytest.mark.skip_ci(reason="Architecture: Only tiny model is suggested")
@require_gptqmodel
def test_autoawq_format_fp_qsave_layers(self):
layer_config = {
diff --git a/test/unit/test_cuda/export/test_auto_gptq_format.py b/test/unit/test_cuda/export/test_auto_gptq_format.py
index b5ca05564..17f93b61d 100644
--- a/test/unit/test_cuda/export/test_auto_gptq_format.py
+++ b/test/unit/test_cuda/export/test_auto_gptq_format.py
@@ -51,7 +51,7 @@ class TestAutoRound:
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto", trust_remote_code=True)
assert model is not None, "Loaded model should not be None."
- @pytest.mark.skip_ci(reason="Only tiny model is suggested") # skip this test in CI
+ @pytest.mark.skip_ci(reason="Architecture: Only tiny model is suggested") # skip this test in CI
@require_gptqmodel
def test_autogptq_format_qsave_ignore_layers(self):
model = AutoModelForCausalLM.from_pretrained(get_model_path("facebook/opt-125m"))
diff --git a/test/unit/test_cuda/export/test_auto_round_format.py b/test/unit/test_cuda/export/test_auto_round_format.py
index b5c0a57cf..9eea2b23a 100644
--- a/test/unit/test_cuda/export/test_auto_round_format.py
+++ b/test/unit/test_cuda/export/test_auto_round_format.py
@@ -18,6 +18,15 @@ from ...envs import (
)
+def _fp8_calibration_data():
+ return [
+ {
+ "input_ids": torch.ones((1, 2), dtype=torch.long),
+ "attention_mask": torch.ones((1, 2), dtype=torch.long),
+ }
+ ]
+
+
class TestAutoRound:
@pytest.fixture(autouse=True, scope="class")
@@ -96,7 +105,7 @@ class TestAutoRound:
)
assert isinstance(model, torch.nn.Module), "Loaded model is not an instance of torch.nn.Module"
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Time-consuming; Accuracy evaluation")
@require_autogptq
def test_mixed_precision(self):
model_name = get_model_path("facebook/opt-125m")
@@ -115,7 +124,7 @@ class TestAutoRound:
eval_generated_prompt(quantized_model_path)
evaluate_accuracy(quantized_model_path, threshold=0.32, batch_size=16)
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Time-consuming; Accuracy evaluation")
@require_gptqmodel
def test_awq_backend(self):
model_name = get_model_path("facebook/opt-125m")
@@ -156,7 +165,7 @@ class TestAutoRound:
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
eval_generated_prompt(model, tokenizer)
- @pytest.mark.skip_ci(reason="Time-consuming; Accuracy evaluation")
+ @pytest.mark.skip_ci(reason="Accuracy: Time-consuming; Accuracy evaluation")
@require_greater_than_050
def test_tritonv2_bf16(self):
model_name = get_model_path("OPEA/Meta-Llama-3.1-8B-Instruct-int4-sym-inc")
@@ -169,13 +178,15 @@ class TestAutoRound:
torch.cuda.empty_cache()
@pytest.mark.timeout(120)
- def test_fp8_block_fp8_format(self):
- model_name = "Qwen/Qwen3-0.6B"
+ def test_fp8_block_fp8_format(self, tiny_qwen_model_path):
+ model_name = tiny_qwen_model_path
scheme = "FP8_BLOCK"
autoround = AutoRound(
model_name,
scheme=scheme,
+ dataset=_fp8_calibration_data(),
+ nsamples=1,
iters=2,
seqlen=2,
)
@@ -194,8 +205,8 @@ class TestAutoRound:
# if is_cuda_support_fp8():
# eval_generated_prompt(quantized_model_path, device="cuda:0")
- def test_fp8_block_autoround_format(self):
- model_name = "Qwen/Qwen3-0.6B"
+ def test_fp8_block_autoround_format(self, tiny_qwen_model_path):
+ model_name = tiny_qwen_model_path
autoround = AutoRound(
model_name,
diff --git a/test/unit/test_cuda/export/test_autoround_int_export.py b/test/unit/test_cuda/export/test_autoround_int_export.py
index aeace8e8b..cc39e2b0f 100644
--- a/test/unit/test_cuda/export/test_autoround_int_export.py
+++ b/test/unit/test_cuda/export/test_autoround_int_export.py
@@ -16,7 +16,7 @@ import shutil
import pytest
import torch
-from transformers import AutoModelForCausalLM
+from transformers import AutoModelForCausalLM, AutoRoundConfig
from auto_round import AutoRound
@@ -31,6 +31,22 @@ def _has_packed_weight(model):
return False
+def _load_with_torch_backend(quantized_model_path):
+ """Reload generic INT exports with the deterministic reference backend.
+
+ ``backend=auto`` can select Marlin, ExLlamaV2, or Triton and compile a
+ backend-specific kernel. Those paths are covered by the dedicated CUDA
+ backend tests; this parameter matrix verifies only export/reload/forward
+ compatibility and must not depend on an incidental optimized backend.
+ """
+ return AutoModelForCausalLM.from_pretrained(
+ quantized_model_path,
+ device_map="cuda:0",
+ trust_remote_code=True,
+ quantization_config=AutoRoundConfig(backend="torch"),
+ )
+
+
class TestAutoroundIntExportGpu:
@pytest.fixture(autouse=True)
def _save_dir(self, tmp_path):
@@ -54,7 +70,7 @@ class TestAutoroundIntExportGpu:
)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = _load_with_torch_backend(quantized_model_path)
assert isinstance(model, torch.nn.Module)
assert _has_packed_weight(model)
input_ids = torch.randint(0, 1000, (1, 16), device="cuda:0")
@@ -106,7 +122,7 @@ class TestAutoroundIntExportGpu:
)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = _load_with_torch_backend(quantized_model_path)
assert isinstance(model, torch.nn.Module)
assert _has_packed_weight(model)
input_ids = torch.randint(0, 1000, (1, 16), device="cuda:0")
@@ -129,6 +145,6 @@ class TestAutoroundIntExportGpu:
)
_, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="auto_round")
- model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="cuda:0", trust_remote_code=True)
+ model = _load_with_torch_backend(quantized_model_path)
assert isinstance(model, torch.nn.Module)
assert _has_packed_weight(model)
diff --git a/test/unit/test_cuda/export/test_gguf_format.py b/test/unit/test_cuda/export/test_gguf_format.py
index ffa8d7562..3cd83f0d8 100644
--- a/test/unit/test_cuda/export/test_gguf_format.py
+++ b/test/unit/test_cuda/export/test_gguf_format.py
@@ -99,6 +99,9 @@ class TestAutoRound:
assert all(tensor.tensor_type == gguf.GGMLQuantizationType.Q4_0 for tensor in tensors)
return gguf_path
+ @pytest.mark.skip_ci(
+ reason="Architecture: Qwen3.5 fused-MoE GGUF export writes 3D expert tensors and requires a large fixture"
+ )
@pytest.mark.skipif(not check_version("transformers>=5.2.0"), reason="requires transformers >= 5.2.0")
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@require_gguf
@@ -139,7 +142,7 @@ class TestAutoRound:
quantized_model_path = self.save_dir
autoround.save_quantized(output_dir=quantized_model_path, format="gguf:q4_1")
- @pytest.mark.skip_ci(reason="Time-consuming accuracy evaluation; covered by nightly")
+ @pytest.mark.skip_ci(reason="Accuracy: Time-consuming accuracy evaluation; covered by nightly")
@require_gguf
def test_q4_0_accuracy(self):
model_name = get_model_path("Qwen/Qwen2.5-0.5B-Instruct")
@@ -159,7 +162,7 @@ class TestAutoRound:
evaluate_accuracy(model, autoround.tokenizer, threshold=0.54, batch_size=16, task="piqa")
- @pytest.mark.skip_ci(reason="Not necessary to test all options in CI")
+ @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all options in CI")
@require_gguf
def test_q2_k_export(self, dataloader):
bits, group_size, sym = 2, 16, False
@@ -186,7 +189,7 @@ class TestAutoRound:
print(result)
shutil.rmtree(tiny_model_path, ignore_errors=True)
- @pytest.mark.skip_ci(reason="Not necessary to test all options in CI")
+ @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all options in CI")
@require_gguf
def test_all_format(self):
for model_name in ["Qwen/Qwen3-8B", "meta-llama/Llama-3.2-3B"]:
@@ -202,7 +205,7 @@ class TestAutoRound:
shutil.rmtree(tiny_model_path, ignore_errors=True)
- @pytest.mark.skip_ci(reason="Not necessary to test special models in CI")
+ @pytest.mark.skip_ci(reason="Architecture: Not necessary to test special models in CI")
@require_gguf
def test_special_model(self):
from test.helpers import save_tiny_model
@@ -224,6 +227,9 @@ class TestAutoRound:
assert abs(file_size - 307) < 5.0
shutil.rmtree(tiny_model_path, ignore_errors=True)
+ @pytest.mark.skip_ci(
+ reason="Architecture: Gemma VLM GGUF export validates mmproj output and requires a large multimodal fixture"
+ )
@require_gguf
@pytest.mark.timeout(120)
def test_vlm_gguf(self):
@@ -268,7 +274,7 @@ class TestAutoRound:
shutil.rmtree(tiny_model_path, ignore_errors=True)
- @pytest.mark.skip_ci(reason="Not necessary to test all options in CI")
+ @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all options in CI")
def test_q2k_mixed(self):
model_path = get_model_path("miromind-ai/MiroThinker-v1.5-30B")
saved_tiny_model_path = save_tiny_model(
@@ -299,6 +305,9 @@ class TestAutoRound:
shutil.rmtree(saved_tiny_model_path, ignore_errors=True)
+ @pytest.mark.skip_ci(
+ reason="Matrix: The Q2_K_S first-eighth FFN-down dtype rule is covered by a dedicated GGUF dtype unit test"
+ )
@require_gguf
@pytest.mark.timeout(90)
def test_q2_k_s_ffn_down_q4k(self):
@@ -329,7 +338,7 @@ class TestAutoRound:
assert ffn_down_type == "Q4_K", f"Expected Q4_K for blk.0.ffn_down.weight but got {ffn_down_type}"
shutil.rmtree(tiny_model_path, ignore_errors=True)
- @pytest.mark.skip_ci(reason="Only tiny model is suggested for CI")
+ @pytest.mark.skip_ci(reason="Architecture: Only tiny model is suggested for CI")
def test_gguf_baseline(self):
model_name = get_model_path("Qwen/Qwen2.5-1.5B-Instruct")
autoround = AutoRound(
diff --git a/test/unit/test_cuda/export/test_llmc_format.py b/test/unit/test_cuda/export/test_llmc_format.py
index 45e71b7f2..1d3443d3b 100644
--- a/test/unit/test_cuda/export/test_llmc_format.py
+++ b/test/unit/test_cuda/export/test_llmc_format.py
@@ -33,7 +33,7 @@ class TestAutoRound:
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
- @pytest.mark.skip_ci(reason="Cannot test all case in CI; time-consuming")
+ @pytest.mark.skip_ci(reason="Matrix: Cannot test all case in CI; time-consuming")
def test_fp8input_mxfp4_llmcompressor_format(self, dataloader, tiny_fp8_qwen_model_path, mock_fp8_capable_device):
scheme = "mxfp4"
ar = AutoRound(
@@ -60,7 +60,7 @@ class TestAutoRound:
and quantization_config["config_groups"]["group_0"]["weights"]["num_bits"] == 4
), f"Invalid MXFP4 quantization configuration: {quantization_config}"
- @pytest.mark.skip_ci(reason="Cannot test all case in CI; time-consuming")
+ @pytest.mark.skip_ci(reason="Matrix: Cannot test all case in CI; time-consuming")
def test_nvfp4_llmcompressor_format(self, tiny_opt_model_path, dataloader):
scheme = "nvfp4"
autoround = AutoRound(
@@ -91,7 +91,7 @@ class TestAutoRound:
and quantization_config["config_groups"]["group_0"]["input_activations"]["num_bits"] == 4
), f"Invalid NVFP4 quantization configuration: {quantization_config}"
- @pytest.mark.skip_ci(reason="Cannot test all case in CI; time-consuming")
+ @pytest.mark.skip_ci(reason="Matrix: Cannot test all case in CI; time-consuming")
def test_fp8_block_llm_compressor_format(self, tiny_qwen_model_path, dataloader):
model_name = get_model_path("Qwen/Qwen3-0.6B")
diff --git a/test/unit/test_cuda/models/test_diffusion.py b/test/unit/test_cuda/models/test_diffusion.py
index 5c402b3c8..3fa81319b 100644
--- a/test/unit/test_cuda/models/test_diffusion.py
+++ b/test/unit/test_cuda/models/test_diffusion.py
@@ -95,7 +95,7 @@ class TestAutoRound:
# skip model saving since it takes much time
autoround.quantize_and_save(tmp_path)
- @pytest.mark.skip_ci(reason="Download large model; Time-consuming")
+ @pytest.mark.skip_ci(reason="Architecture: Download large model; Time-consuming")
def test_diffusion_model_checker(self):
from auto_round.utils import is_diffusion_model
@@ -105,7 +105,7 @@ class TestAutoRound:
assert is_diffusion_model(get_model_path("Qwen/Qwen3-8B")) is False
@multi_card
- @pytest.mark.skip_ci(reason="multiple card test")
+ @pytest.mark.skip_ci(reason="Resource: multiple card test")
def test_diffusion_tune_on_multi_cards(self, tiny_flux_model_path, tmp_path):
from diffusers import AutoPipelineForText2Image
diff --git a/test/unit/test_cuda/models/test_fp8_model.py b/test/unit/test_cuda/models/test_fp8_model.py
index 8829f77ad..73f64b223 100644
--- a/test/unit/test_cuda/models/test_fp8_model.py
+++ b/test/unit/test_cuda/models/test_fp8_model.py
@@ -96,7 +96,7 @@ class TestAutoRound:
# inputs = tokenizer(text, return_tensors="pt").to(model.device)
# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
- @pytest.mark.skip_ci(reason="time-consuming")
+ @pytest.mark.skip_ci(reason="Architecture: time-consuming")
def test_small_model_rtn(self, mock_fp8_capable_device):
model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")
ar = AutoRound(model=model_name, iters=0, disable_opt_rtn=True)
@@ -104,7 +104,7 @@ class TestAutoRound:
with patch("torch.cuda.get_device_capability", return_value=DEVICE_CAPABILITY): # revert DEVICE_CAPABILITY
evaluate_accuracy(folder, threshold=0.25)
- @pytest.mark.skip_ci(reason="time-consuming")
+ @pytest.mark.skip_ci(reason="Architecture: time-consuming")
def test_small_model_iters1(self, mock_fp8_capable_device):
model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")
ar = AutoRound(model=model_name, iters=1)
@@ -120,7 +120,7 @@ class TestAutoRound:
with patch("torch.cuda.get_device_capability", return_value=DEVICE_CAPABILITY): # revert DEVICE_CAPABILITY
evaluate_accuracy(folder, threshold=0.33)
- @pytest.mark.skip_ci(reason="time-consuming")
+ @pytest.mark.skip_ci(reason="Architecture: time-consuming")
def test_small_model_rtn_with_lm_head(self, mock_fp8_capable_device):
model_name = get_model_path("Qwen/Qwen3-0.6B-FP8")
layer_config = {"lm_head": {"bits": 4}}
@@ -141,7 +141,7 @@ class TestAutoRound:
output = llm("There is a girl who likes adventure,", max_tokens=32)
print(output)
- @pytest.mark.skip_ci(reason="Not necessary to test all options in CI")
+ @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all options in CI")
def test_fp8_model_gguf_q3(self, mock_fp8_capable_device, tiny_fp8_qwen_model_path):
from llama_cpp import Llama
@@ -154,7 +154,7 @@ class TestAutoRound:
output = llm("There is a girl who likes adventure,", max_tokens=32)
print(output)
- @pytest.mark.skip_ci(reason="Not necessary to test all options in CI")
+ @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all options in CI")
@pytest.mark.parametrize("scheme", ["MXFP4", "NVFP4"])
def test_diff_datatype(self, scheme, tiny_fp8_qwen_model_path, mock_fp8_capable_device):
model_name = tiny_fp8_qwen_model_path
diff --git a/test/unit/test_cuda/models/test_get_block_name.py b/test/unit/test_cuda/models/test_get_block_name.py
index bfeb03133..633b797c2 100644
--- a/test/unit/test_cuda/models/test_get_block_name.py
+++ b/test/unit/test_cuda/models/test_get_block_name.py
@@ -19,7 +19,7 @@ from auto_round import AutoRound
from auto_round.utils import get_block_names, is_pure_text_model
[email protected]_ci(reason="Only tiny model is suggested")
[email protected]_ci(reason="Architecture: Only tiny model is suggested")
class TestAutoRound:
@classmethod
def setup_class(self):
diff --git a/test/unit/test_cuda/models/test_mllm.py b/test/unit/test_cuda/models/test_mllm.py
index ef710ff3a..c91b99325 100644
--- a/test/unit/test_cuda/models/test_mllm.py
+++ b/test/unit/test_cuda/models/test_mllm.py
@@ -37,7 +37,7 @@ class VisionDataLoader:
}
[email protected]_ci(reason="Only tiny model is suggested")
[email protected]_ci(reason="Architecture: Only tiny model is suggested")
class TestAutoRound:
@pytest.fixture(autouse=True)
diff --git a/test/unit/test_cuda/models/test_moe_model.py b/test/unit/test_cuda/models/test_moe_model.py
index 04c37a319..fe3e3e34f 100644
--- a/test/unit/test_cuda/models/test_moe_model.py
+++ b/test/unit/test_cuda/models/test_moe_model.py
@@ -1,35 +1,68 @@
-import shutil
from test.helpers import check_version
import pytest
import torch
-from transformers import AutoTokenizer, Llama4ForConditionalGeneration
-from transformers.models.gpt_oss.modeling_gpt_oss import GptOssForCausalLM
-from transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe import Qwen3VLMoeForConditionalGeneration
+from transformers import AutoRoundConfig
from auto_round import AutoRound
+def _calibration_data(seqlen):
+ return [
+ {
+ "input_ids": torch.ones((1, seqlen), dtype=torch.long),
+ "attention_mask": torch.ones((1, seqlen), dtype=torch.bool),
+ }
+ ]
+
+
[email protected](not check_version("transformers>=5.2.0"), reason="requires transformers >= 5.2.0")
+def test_qwen3_5_moe(tiny_qwen35_moe_text_model_path, tmp_path):
+ from transformers import Qwen3_5MoeForCausalLM
+
+ ar = AutoRound(
+ tiny_qwen35_moe_text_model_path,
+ dataset=_calibration_data(8),
+ nsamples=1,
+ seqlen=8,
+ iters=1,
+ # The tiny expert down projection is 64-wide; use a supported group size instead of fake fallback.
+ group_size=32,
+ )
+ quantized_model, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=tmp_path / "quantized")
+ assert quantized_model is not None, "Quantized model should not be None."
+
+ # Small tensors can select a fake backend that has no CUDA kernel; optimized backend selection has dedicated CUDA tests.
+ loaded_model = Qwen3_5MoeForCausalLM.from_pretrained(
+ quantized_model_path, quantization_config=AutoRoundConfig(backend="torch")
+ ).to("cuda")
+ inp = torch.randint(0, loaded_model.config.vocab_size, (1, 8), device="cuda")
+ with torch.inference_mode():
+ loaded_out = loaded_model(inp)
+ assert torch.isfinite(loaded_out.logits).all()
+
+
+ reason="Architecture: Conditional Qwen3.5 fused-MoE validates the large VLM fixture; retain in the full test tier"
+)
@pytest.mark.skipif(not check_version("transformers>=5.2.0"), reason="requires transformers >= 5.2.0")
@pytest.mark.timeout(300)
-def test_qwen3_5_moe(tiny_qwen35_moe_model_path):
+def test_qwen3_5_moe_conditional(tiny_qwen35_moe_model_path, tmp_path):
from transformers import Qwen3_5MoeForConditionalGeneration
- output_dir = "test_quantized_qwen35_moe"
ar = AutoRound(
tiny_qwen35_moe_model_path,
+ dataset=_calibration_data(32),
nsamples=2,
seqlen=32,
iters=1,
)
- quantized_model, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=output_dir)
+ quantized_model, quantized_model_path = ar.quantize_and_save(format="auto_round", output_dir=tmp_path / "quantized")
assert quantized_model is not None, "Quantized model should not be None."
- loaded_model = Qwen3_5MoeForConditionalGeneration.from_pretrained(quantized_model_path)
- loaded_model.to("cuda")
-
- inp = torch.randint(0, 100, (1, 64)).to("cuda")
+ loaded_model = Qwen3_5MoeForConditionalGeneration.from_pretrained(quantized_model_path).to("cuda")
+ inp = torch.randint(0, loaded_model.config.text_config.vocab_size, (1, 64), device="cuda")
with torch.inference_mode():
loaded_out = loaded_model(inp)
-
- shutil.rmtree(output_dir, ignore_errors=True)
+ assert torch.isfinite(loaded_out.logits).all()
diff --git a/test/unit/test_cuda/models/test_omni_model.py b/test/unit/test_cuda/models/test_omni_model.py
index 85cacee4c..99559ead3 100644
--- a/test/unit/test_cuda/models/test_omni_model.py
+++ b/test/unit/test_cuda/models/test_omni_model.py
@@ -31,14 +31,51 @@ from test.helpers import check_version
import pytest
import torch
from transformers import (
+ AutoRoundConfig,
AutoTokenizer,
Qwen2_5OmniForConditionalGeneration,
- Qwen3OmniMoeConfig,
Qwen3OmniMoeForConditionalGeneration,
)
from auto_round import AutoRound
+
+def _calibration_data(seqlen):
+ return [
+ {
+ "input_ids": torch.ones((1, seqlen), dtype=torch.long),
+ "attention_mask": torch.ones((1, seqlen), dtype=torch.long),
+ }
+ ]
+
+
+def _make_text_only_autoround(model_path, tokenizer_path, scheme="W4A16"):
+ model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(model_path)
+ tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
+ return AutoRound(
+ model,
+ tokenizer=tokenizer,
+ # Pre-tokenized calibration bypasses processor encoding; reuse the serializable tokenizer for MLLM lifecycle.
+ processor=tokenizer,
+ dataset=_calibration_data(8),
+ scheme=scheme,
+ nsamples=1,
+ iters=1,
+ seqlen=8,
+ group_size=32,
+ # This smoke uses pre-tokenized text inputs; the Qwen3-Omni template and processor are covered in common tests.
+ template="default",
+ ignore_layers="self_attn,lm_head,mlp.gate",
+ )
+
+
+def _load_with_torch_backend(save_folder):
+ # Small smoke tensors can select a JIT backend; optimized backend selection has dedicated CUDA tests.
+ return Qwen3OmniMoeForConditionalGeneration.from_pretrained(
+ save_folder, quantization_config=AutoRoundConfig(backend="torch")
+ ).to("cuda")
+
+
pytestmark = [
pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available"),
pytest.mark.skipif(
@@ -51,7 +88,7 @@ pytestmark = [
# ========================= Qwen2.5-Omni Integration Tests ==================
[email protected]_ci(reason="Not necessary to test all cases in CI")
[email protected]_ci(reason="Matrix: Not necessary to test all cases in CI")
class TestQwen2_5OmniQuantization:
"""End-to-end quantization test for Qwen2.5-Omni (dense model)."""
@@ -97,44 +134,27 @@ class TestQwen3OmniMoeQuantization:
shutil.rmtree(self.save_dir, ignore_errors=True)
@pytest.mark.timeout(180)
- def test_quantize_and_reload(self, tiny_qwen3_omni_moe_model_path):
- """Quantize, save, reload, verify weights, and run inference."""
- # Quantize
- autoround = AutoRound(
- tiny_qwen3_omni_moe_model_path,
- nsamples=2,
- iters=1,
- seqlen=32,
- ignore_layers="self_attn,lm_head,mlp.gate",
- )
+ def test_quantize_and_reload(self, tiny_opt_model_path, tiny_qwen3_omni_moe_model_path):
+ """Quantize, save, reload, and run Qwen3-Omni's text-only thinker path."""
+ autoround = _make_text_only_autoround(tiny_qwen3_omni_moe_model_path, tiny_opt_model_path)
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=self.save_dir)
assert quantized_model is not None, "Quantized model should not be None"
- # Reload
- loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(save_folder, device_map="cuda")
-
- # Run inference on thinker
- inp = torch.randint(0, 100, (1, 64)).to("cuda")
+ loaded_model = _load_with_torch_backend(save_folder)
+ inp = torch.randint(0, 100, (1, 8), device="cuda")
with torch.inference_mode():
output = loaded_model.thinker(input_ids=inp)
- assert output is not None, "Inference failed on reloaded model (thinker)"
+ assert torch.isfinite(output.logits).all(), "Reloaded Qwen3-Omni thinker produced non-finite logits"
- @pytest.mark.skip_ci(reason="Not necessary to test all schemes in CI")
- def test_quantize_mxfp4(self, tiny_qwen3_omni_moe_model_path):
+ @pytest.mark.skip_ci(reason="Matrix: Not necessary to test all schemes in CI")
+ def test_quantize_mxfp4(self, tiny_opt_model_path, tiny_qwen3_omni_moe_model_path):
"""Quantize with MXFP4 scheme and verify."""
- autoround = AutoRound(
- tiny_qwen3_omni_moe_model_path,
- scheme="MXFP4",
- nsamples=2,
- iters=1,
- seqlen=32,
- ignore_layers="self_attn,lm_head,mlp.gate",
- )
+ autoround = _make_text_only_autoround(tiny_qwen3_omni_moe_model_path, tiny_opt_model_path, scheme="MXFP4")
quantized_model, save_folder = autoround.quantize_and_save(format="auto_round", output_dir=self.save_dir)
assert quantized_model is not None, "MXFP4 quantized model should not be None"
# Reload and inference
- loaded_model = Qwen3OmniMoeForConditionalGeneration.from_pretrained(save_folder, device_map="cuda")
+ loaded_model = _load_with_torch_backend(save_folder)
inp = torch.randint(0, 100, (1, 64)).to("cuda")
with torch.inference_mode():
diff --git a/test/unit/test_cuda/models/test_support_vlms.py b/test/unit/test_cuda/models/test_support_vlms.py
index cfff833e1..e248cf689 100644
--- a/test/unit/test_cuda/models/test_support_vlms.py
+++ b/test/unit/test_cuda/models/test_support_vlms.py
@@ -16,7 +16,7 @@ AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
[email protected]_ci(reason="Only tiny model is suggested")
[email protected]_ci(reason="Architecture: Only tiny model is suggested")
class TestSupportVLMS:
@classmethod
diff --git a/test/unit/test_cuda/quantization/test_mxfp_nvfp.py b/test/unit/test_cuda/quantization/test_mxfp_nvfp.py
index 25c3c54cd..e214bf48b 100644
--- a/test/unit/test_cuda/quantization/test_mxfp_nvfp.py
+++ b/test/unit/test_cuda/quantization/test_mxfp_nvfp.py
@@ -111,7 +111,7 @@ class TestAutoRound:
print(model)
assert model is not None, "Failed to load the quantized model."
- @pytest.mark.skip_ci(reason="Cannot test all case in CI; time-consuming")
+ @pytest.mark.skip_ci(reason="Matrix: Cannot test all cases in CI; time-consuming")
def test_nvfp4_moe_actmax_ar(self, tiny_deepseek_v2_model_path, dataloader):
scheme = "nvfp4"
autoround = AutoRound(
diff --git a/test/unit/test_cuda/quantization/test_torch_compile.py b/test/unit/test_cuda/quantization/test_torch_compile.py
index 6c6cd4ecb..6ea92ec5f 100644
--- a/test/unit/test_cuda/quantization/test_torch_compile.py
+++ b/test/unit/test_cuda/quantization/test_torch_compile.py
@@ -27,7 +27,7 @@ class TestTorchCompile:
shutil.rmtree("./saved", ignore_errors=True)
shutil.rmtree("runs", ignore_errors=True)
- @pytest.mark.skip_ci(reason="torch.compile warmup with iters=2 is slow")
+ @pytest.mark.skip_ci(reason="Backend/JIT: torch.compile warmup with iters=2 is slow")
@require_gguf
def test_gguf_q2ks_torch_compile(self, dataloader):
"""Test GGUF Q2_K_S quantization with torch.compile enabled.
diff --git a/test/unit/test_cuda/transform/test_mxfp4_transform.py b/test/unit/test_cuda/transform/test_mxfp4_transform.py
index b3ee169aa..7bd5b30cf 100644
--- a/test/unit/test_cuda/transform/test_mxfp4_transform.py
+++ b/test/unit/test_cuda/transform/test_mxfp4_transform.py
@@ -46,6 +46,9 @@ class TestAutoRound:
generate_prompt(model, tokenizer)
+ @pytest.mark.skip_ci(
+ reason="Matrix: Default MXFP4 transform covers CUDA export/reload; MXFP4 tuning remains in the full test tier"
+ )
@pytest.mark.timeout(90)
def test_transform_mxfp4_tuning_quant_infer(self):
model_name = get_model_path("qwen/Qwen3-0.6B")
@@ -66,6 +69,9 @@ class TestAutoRound:
generate_prompt(model, tokenizer)
+ @pytest.mark.skip_ci(
+ reason="Matrix: Random Hadamard behavior has dedicated unit coverage; keep the CUDA MXFP4 E2E smoke deterministic"
+ )
@pytest.mark.timeout(90)
def test_random_transform_mxfp4_quant_infer(self):
model_name = get_model_path("qwen/Qwen3-0.6B")
diff --git a/test/unit/test_cuda/transform/test_spinquant.py b/test/unit/test_cuda/transform/test_spinquant.py
index 570691046..815602752 100644
--- a/test/unit/test_cuda/transform/test_spinquant.py
+++ b/test/unit/test_cuda/transform/test_spinquant.py
@@ -26,6 +26,11 @@ from auto_round.algorithms.transforms.spinquant.preprocessor import (
remove_spinquant_hooks_from_model,
)
+_PAIRWISE_FULL_CI = pytest.mark.skip_ci(
+ reason="Matrix: Full deterministic rotation already covers pairwise interactions"
+)
+_RANDOM_SINGLE_FULL_CI = pytest.mark.skip_ci(reason="Matrix: All-random rotation covers every random rotation path")
+
# ═══════════════════════════════════════════════════════════════════════════════
# Config Tests
# ═══════════════════════════════════════════════════════════════════════════════
@@ -345,6 +350,9 @@ class TestPipelineIntegration:
)
shutil.rmtree(self.save_dir + "_cfg", ignore_errors=True)
+ @pytest.mark.skip_ci(
+ reason="Matrix: The SpinQuantConfig-object pipeline is already covered by test_pipeline_spinquant_config"
+ )
@pytest.mark.timeout(90)
def test_pipeline_config_object(self):
"""Building a SpinQuantConfig directly (instead of the dict shorthand) and passing
@@ -469,18 +477,26 @@ class TestRotationEquivalence:
(False, True, False, False, False, False, False, False, "R2"),
(False, False, True, False, False, False, False, False, "R3"),
(False, False, False, True, False, False, False, False, "R4"),
- # --- Pairwise combinations (deterministic) ---
- (True, True, False, False, False, False, False, False, "R1+R2"),
- (True, False, True, False, False, False, False, False, "R1+R3"),
- (True, False, False, True, False, False, False, False, "R1+R4"),
- (False, False, True, True, False, False, False, False, "R3+R4"),
+ # --- Pairwise combinations (deterministic), retained for full CI ---
+ pytest.param(True, True, False, False, False, False, False, False, "R1+R2", marks=_PAIRWISE_FULL_CI),
+ pytest.param(True, False, True, False, False, False, False, False, "R1+R3", marks=_PAIRWISE_FULL_CI),
+ pytest.param(True, False, False, True, False, False, False, False, "R1+R4", marks=_PAIRWISE_FULL_CI),
+ pytest.param(False, False, True, True, False, False, False, False, "R3+R4", marks=_PAIRWISE_FULL_CI),
# --- Full combination (deterministic) ---
(True, True, True, True, False, False, False, False, "R1+R2+R3+R4"),
- # --- Random Hadamard variants ---
- (True, False, False, False, True, False, False, False, "R1-random"),
- (False, True, False, False, False, True, False, False, "R2-random"),
- (False, False, True, False, False, False, True, False, "R3-random"),
- (False, False, False, True, False, False, False, True, "R4-random"),
+ # --- Random Hadamard variants, retained for full CI ---
+ pytest.param(
+ True, False, False, False, True, False, False, False, "R1-random", marks=_RANDOM_SINGLE_FULL_CI
+ ),
+ pytest.param(
+ False, True, False, False, False, True, False, False, "R2-random", marks=_RANDOM_SINGLE_FULL_CI
+ ),
+ pytest.param(
+ False, False, True, False, False, False, True, False, "R3-random", marks=_RANDOM_SINGLE_FULL_CI
+ ),
+ pytest.param(
+ False, False, False, True, False, False, False, True, "R4-random", marks=_RANDOM_SINGLE_FULL_CI
+ ),
(True, True, True, True, True, True, True, True, "R1+R2+R3+R4-all-random"),
],
)
diff --git a/test/unit/test_xpu/backends/test_torch_backend.py b/test/unit/test_xpu/backends/test_torch_backend.py
index f8af431fc..5ade5afe2 100644
--- a/test/unit/test_xpu/backends/test_torch_backend.py
+++ b/test/unit/test_xpu/backends/test_torch_backend.py
@@ -82,7 +82,9 @@ class TestTorchBackendFunctional:
tokenizer = AutoTokenizer.from_pretrained(self.save_dir)
model_infer(model, tokenizer)
- @pytest.mark.skip_ci(reason="Not necessary to test both symmetric and asymmetric for 3-bit quantization in CI")
+ @pytest.mark.skip_ci(
+ reason="Matrix: Not necessary to test both symmetric and asymmetric for 3-bit quantization in CI"
+ )
@pytest.mark.parametrize("device", _AVAILABLE_DEVICES)
def test_torch_backend_3bit_sym(self, tiny_opt_model_path, dataloader, device):
"""3-bit symmetric model loads via the torch backend and generates text."""