PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add workaround for llmc issue and add unit tests for model_free_utils helpers (#2207)
Xin Hea2c27b019160
diff --git a/auto_round/utils/model_free_utils.py b/auto_round/utils/model_free_utils.pyindex 4ec86e902..0aa968938 100644--- a/auto_round/utils/model_free_utils.py+++ b/auto_round/utils/model_free_utils.py@@ -1455,6 +1455,30 @@ def _build_nvfp4_e5m3_quantization_config(ignored_layers: list[str]) -> dict:return qconfig+# TODO: Remove when the issue is fixed+# https://github.com/vllm-project/llm-compressor/issues/3069+def _add_routed_experts_if_moe(targets: list[str], layer_names: list[str]) -> list[str]:+ """Append ``"RoutedExperts"`` to *targets* when *layer_names* indicates a MoE model.++ LLM-Compressor has a known bug where MoE expert projections (e.g. ``w1``/+ ``w3`` or layers inside ``block_sparse_moe``) are not matched by the generic+ ``"Linear"`` target and require the explicit ``"RoutedExperts"`` target to be+ present in the config group. This helper detects that situation and injects+ the extra target so quantization is applied correctly.++ A model is considered MoE if any layer name contains:++ * ``.w1`` or ``.w3`` (typical Mixtral / DeepSeek gating projections), or+ * ``block_sparse_moe`` (Mixtral-style naming).+ """+ if "RoutedExperts" in targets:+ return targets+ moe_re = re.compile(r"\.w[13](?:\.|$)|block_sparse_moe")+ if any(moe_re.search(name) for name in layer_names):+ return list(targets) + ["RoutedExperts"]+ return targets++def _get_mxfp_group_scheme_and_format(group_bits: int,group_data_type: str,@@ -1564,7 +1588,8 @@ def _build_mxfp_quantization_config(qconfig = initialize_nvfp4_e5m3_quantization(ignore=ignore)if is_fp_default and scheme_groups:- qconfig["config_groups"]["group_0"]["targets"] = list(quantized_layers)+ targets = _add_routed_experts_if_moe(list(quantized_layers), quantized_layers)+ qconfig["config_groups"]["group_0"]["targets"] = targetsqconfig["format"] = fmtqconfig.update(_get_llm_compressor_metadata())return qconfig@@ -1573,7 +1598,7 @@ def _build_mxfp_quantization_config(scheme_name = "MXFP4" if actual_bits == 4 else "MXFP8"qconfig = _init_q(scheme=scheme_name, ignore=ignore)if is_fp_default and scheme_groups:- targets = list(quantized_layers)+ targets = _add_routed_experts_if_moe(list(quantized_layers), quantized_layers)qconfig.config_groups["group_0"].targets = targetsqconfig = qconfig.to_dict()qconfig["format"] = fmt@@ -1599,6 +1624,7 @@ def _build_mxfp_quantization_config(group_name = f"group_{idx}"is_default_group = (group_bits, group_dt) == default_keytargets = ["Linear"] if is_default_group else layer_names+ targets = _add_routed_experts_if_moe(targets, layer_names)group_scheme, fmt = _get_mxfp_group_scheme_and_format(group_bits, group_dt, ignore)group_scheme.targets = targetsconfig_groups[group_name] = group_scheme@@ -1948,6 +1974,8 @@ def _build_quantization_config(if is_fp_default and _layer_config_has_nvfp4(layer_config) and format == "llm_compressor":nvfp4_dt = _get_layer_config_nvfp4_dt(layer_config)targets = list(dict.fromkeys(quantized_layers)) if quantized_layers else ["Linear"]+ if quantized_layers:+ targets = _add_routed_experts_if_moe(targets, quantized_layers)if nvfp4_dt == _NVFP4_E5M3_DATA_TYPE:warnings.warn("LLMC/llm-compressor does not currently support the NVFP4_E5M3 scheme. "diff --git a/test/unit/test_cpu/quantization/test_model_free.py b/test/unit/test_cpu/quantization/test_model_free.pyindex 2bbf071a0..c4c7ab627 100644--- a/test/unit/test_cpu/quantization/test_model_free.py+++ b/test/unit/test_cpu/quantization/test_model_free.py@@ -54,16 +54,13 @@ from auto_round import AutoRoundfrom auto_round.compressors.model_free import (_ModelFreeCompressorCore,_process_single_shard_task,- get_predefined_ignore_layers_from_config,)from auto_round.schemes import QuantizationScheme-from auto_round.utils.model import is_model_free_routefrom auto_round.utils.model_free_utils import (_build_mxfp_autoround_quantization_config,_build_mxfp_quantization_config,_build_quantization_config,_convert_auto_scheme_layer_config,- _dequant_fp8_tensors,_dequant_mxfp_tensors,_expand_e8m0_block_scale,_handle_mxfp_source_tensors,@@ -73,11 +70,6 @@ from auto_round.utils.model_free_utils import (_quantize_weight_mxfp,_quantize_weight_nvfp4_e5m3,_validate_auto_scheme_options,-)-from auto_round.utils.model_free_utils import (- handle_model_type_low_precision_source_tensors as _handle_model_type_low_precision_source_tensors,-)-from auto_round.utils.model_free_utils import (is_model_free_supported_scheme,)from auto_round.utils.model_free_utils import (@@ -134,18 +126,6 @@ from ...envs import require_compressed_tensors# ---------------------------------------------------------------------------[email protected]("scheme", ["MXFP8", "mxfp8"])-def test_model_free_route_accepts_mxfp_scheme_case_insensitively(scheme):- assert is_model_free_route("model", scheme, 0, True, {})--[email protected]("dtype_key", ["static_kv_dtype", "static_attention_dtype"])[email protected]("explicit", [False, True])-def test_model_free_route_rejects_static_attention_quantization(dtype_key, explicit):- kwargs = {dtype_key: "fp8", "model_free": explicit}- assert not is_model_free_route("model", "W4A16", 0, True, kwargs)--_LLAMA_CFG = {"architectures": ["LlamaForCausalLM"], "model_type": "llama"}_DEFAULT_SCHEME = {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"}@@ -194,10 +174,6 @@ def _make_model_dir(tmp_path, config, tensors, *, multi_shard=False):return model_dir-def _matcher(ignore=None, layer_config=None, default=None):- return _PatternMatcher(ignore or [], layer_config or {}, default or {})--def _read_output_keys(output_dir):keys = set()for f in os.listdir(output_dir):@@ -212,53 +188,6 @@ def _read_qconfig(output_dir):return json.load(f).get("quantization_config", {})-# ===========================================================================-# _PatternMatcher-# ===========================================================================---class TestPatternMatcher:- def test_ignore_substring(self):- m = _matcher(ignore=["mlp"])- assert m.should_ignore("model.layers.0.mlp.fc1.weight") is True- assert m.should_ignore("model.layers.0.self_attn.q_proj.weight") is False-- def test_ignore_trailing_dot(self):- m = _matcher(ignore=["layers.4."])- assert m.should_ignore("model.layers.4.mlp.fc1.weight") is True- assert m.should_ignore("model.layers.45.mlp.fc1.weight") is False-- def test_skip_predefined(self):- m = _matcher()- assert m.should_skip("model.layers.0.shared_expert_gate.weight") is True- assert m.should_skip("model.layers.0.mlp.gate.weight") is True- assert m.should_skip("model.layers.0.mlp.gate_proj.weight") is False- assert m.should_skip("model.embed_tokens.weight") is True- assert m.should_skip("model.layers.0.mlp.fc1.weight") is False-- def test_resolve_scheme_exact_regex_and_default(self):- default = {"bits": 4, "group_size": 128, "sym": True}- lc = {- "model.layers.0.mlp.fc1": {"bits": 8, "group_size": 32},- r".*k_proj": {"bits": 8},- }- m = _matcher(layer_config=lc, default=default)- assert m.resolve_scheme("model.layers.0.mlp.fc1.weight")["bits"] == 8- assert m.resolve_scheme("model.layers.0.self_attn.k_proj.weight")["bits"] == 8- assert m.resolve_scheme("model.layers.0.mlp.fc2.weight") == default-- def test_resolve_bits16_returns_none(self):- m = _matcher(layer_config={"model.layers.0.fc1": {"bits": 16}}, default={"bits": 4, "group_size": 128})- assert m.resolve_scheme("model.layers.0.fc1.weight") is None-- def test_resolve_substring_pattern(self):- default = {"bits": 4, "group_size": 128, "sym": True}- m = _matcher(layer_config={".ffn.experts.": {"bits": 2, "group_size": 64}}, default=default)- r = m.resolve_scheme("model.layers.0.ffn.experts.3.gate_proj.weight")- assert r["bits"] == 2 and r["group_size"] == 64- assert m.resolve_scheme("model.layers.0.self_attn.q_proj.weight") == default--# ===========================================================================# _parse_layer_config — scheme key resolution# ===========================================================================@@ -282,7 +211,11 @@ class TestParseLayerConfig:cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)assert cfg["bits"] == 2 and "scheme" not in cfg- m = _matcher(layer_config=core.layer_config, default=core.default_scheme)+ m = _PatternMatcher(+ ignore_patterns=[],+ layer_config=core.layer_config,+ default_scheme=core.default_scheme,+ )assert m.resolve_scheme("model.layers.0.ffn.experts.3.gate_proj.weight")["bits"] == 2def test_scheme_key_with_overrides(self):@@ -587,94 +520,15 @@ def test_model_free_legacy_nvfp4_is_normalized_and_passthrough(tmp_path):# ===========================================================================-# FP8 source model+# FP8 source model — moved to test/unit/test_cpu/utils/test_model_free_utils.py# ===========================================================================-class TestFP8Source:- def test_dequant_fp8(self):- w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)- raw = {"layer.weight": w, "layer.weight_scale_inv": torch.tensor(0.5), "layer.bias": torch.randn(64)}- result = _dequant_fp8_tensors(raw, block_size=None)- assert result["layer.weight"].dtype == torch.bfloat16 and "layer.weight_scale_inv" not in result-- def test_no_fp8_noop(self):- raw = {"layer.weight": torch.randn(64, 128)}- assert _dequant_fp8_tensors(raw, block_size=None) is raw-- def test_process_shard_fp8(self, tmp_path):- shard_path = str(tmp_path / "shard.safetensors")- w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)- save_file({"layer.weight": w, "layer.weight_scale_inv": torch.tensor(1.0)}, shard_path)- output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [], device="cpu", fp8_block_size=None)- assert "layer" in quantized and "layer.qweight" in output-- def test_dequant_fp8_hydrates_scale_from_sibling_shard(self, tmp_path):- """When scale_inv is sharded separately, dequant should hydrate it via index."""- shard_dir = tmp_path / "source"- shard_dir.mkdir(parents=True, exist_ok=True)-- weight_name = "model.layers.0.mlp.experts.1.gate_proj.weight"- scale_name = "model.layers.0.mlp.experts.1.gate_proj.weight_scale_inv"-- shard_a = shard_dir / "model-00001-of-00002.safetensors"- shard_b = shard_dir / "model-00002-of-00002.safetensors"- save_file({weight_name: torch.randn(2048, 7168, dtype=torch.bfloat16).to(torch.float8_e4m3fn)}, str(shard_a))- save_file({scale_name: torch.ones(16, 56, dtype=torch.float32)}, str(shard_b))-- with open(shard_dir / "model.safetensors.index.json", "w") as f:- json.dump(- {- "metadata": {"total_size": 0},- "weight_map": {- weight_name: shard_a.name,- scale_name: shard_b.name,- },- },- f,- )-- output, quantized, _ = _process_shard(- str(shard_a),- _DEFAULT_SCHEME,- {},- [],- fp8_block_size=[128, 128],- )- assert "model.layers.0.mlp.experts.1.gate_proj" in quantized- assert "model.layers.0.mlp.experts.1.gate_proj.qweight" in output--# ===========================================================================# Quantization config builder# ===========================================================================--class TestBuildQuantizationConfig:- def test_extra_config_filters_embed_conv_only(self):- default = {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"}- ignored = [- "model.embed_tokens",- "model.conv1",- "model.layers.0.shared_expert_gate",- "model.layers.0.mlp.gate",- ]-- cfg = _build_quantization_config(- default_scheme=default,- layer_config={},- ignore_patterns=[],- quantized_layers=[],- ignored_layers=ignored,- )-- extra = cfg.get("extra_config", {})- # Non-Linear ops are filtered out.- assert "model.embed_tokens" not in extra- assert "model.conv1" not in extra- # Other ignored layers should still be recorded.- assert extra["model.layers.0.shared_expert_gate"] == {"bits": 16, "data_type": "float"}- assert extra["model.layers.0.mlp.gate"] == {"bits": 16, "data_type": "float"}+# (TestBuildQuantizationConfig moved to test/unit/test_cpu/utils/test_model_free_utils.py)# ===========================================================================@@ -810,42 +664,11 @@ class TestModelFreeQuantize:class TestModelFreeMXFP:- """End-to-end tests for MXFP4/MXFP8 model-free quantization."""-- def test_disable_opt_rtn_skips_mxfp_scale_search(self, monkeypatch):- from auto_round.data_type import mxfp-- search_mock = Mock(side_effect=AssertionError("optimized RTN must be disabled"))- monkeypatch.setattr(mxfp, "quant_mx_opt_rtn", search_mock)-- weight = torch.randn(8, 32, dtype=torch.bfloat16)- out = _quantize_weight_mxfp(- weight,- "layer",- bits=4,- group_size=32,- data_type="mx_fp",- disable_opt_rtn=True,- )+ """End-to-end tests for MXFP4/MXFP8 model-free quantization.- assert "layer.weight_packed" in out- search_mock.assert_not_called()-- def test_quantize_weight_mxfp4_shapes(self):- w = torch.randn(64, 128, dtype=torch.bfloat16)- out = _quantize_weight_mxfp(w, "layer", bits=4, group_size=32, data_type="mx_fp")- assert out["layer.weight_packed"].shape == (64, 64) # in_features / 2- assert out["layer.weight_packed"].dtype == torch.uint8- assert out["layer.weight_scale"].shape == (64, 4) # in_features / group_size- assert out["layer.weight_scale"].dtype == torch.uint8-- def test_quantize_weight_mxfp8_shapes(self):- w = torch.randn(64, 128, dtype=torch.bfloat16)- out = _quantize_weight_mxfp(w, "layer", bits=8, group_size=32, data_type="mx_fp")- assert out["layer.weight"].shape == (64, 128)- assert out["layer.weight"].dtype == torch.float8_e4m3fn- assert out["layer.weight_scale"].shape == (64, 4)- assert out["layer.weight_scale"].dtype == torch.uint8+ Unit tests for _quantize_weight_mxfp and _build_mxfp_quantization_config+ have been moved to test/unit/test_cpu/utils/test_model_free_utils.py.+ """@require_compressed_tensors@pytest.mark.parametrize("scheme,fmt", [("MXFP4", "mxfp4-pack-quantized")])@@ -896,61 +719,8 @@ class TestModelFreeMXFP:assert "layer.fc1.weight_packed" in outputassert "layer.fc1.weight_scale" in output- @require_compressed_tensors- def test_build_mxfp_mixed_config_uniform(self):- """Single-scheme path: no layer_config overrides → uniform format."""- default = {"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"}- quantized = ["model.layers.0.fc1", "model.layers.0.fc2"]- ignored = ["lm_head"]- cfg = _build_mxfp_quantization_config(default, quantized, ignored, layer_config={})- assert cfg["format"] == "mxfp4-pack-quantized"- assert "lm_head" in cfg["ignore"]- assert len(cfg["config_groups"]) == 1-- @require_compressed_tensors- def test_build_mxfp_config_bf16_default_targets_only_overrides(self):- """A single MXFP override must not describe all BF16 Linear layers as quantized."""- default = {"bits": 16, "group_size": -1, "sym": True, "data_type": "bf16"}- layer_config = {- "model.layers.0.self_attn.q_proj": {"bits": 4, "group_size": 32, "data_type": "mx_fp"},- }- quantized = ["model.layers.0.self_attn.q_proj"]-- cfg = _build_mxfp_quantization_config(default, quantized, [], layer_config=layer_config)-- assert cfg["format"] == "mxfp4-pack-quantized"- assert cfg["config_groups"]["group_0"]["targets"] == quantized-- @require_compressed_tensors- def test_build_mxfp_mixed_config_two_groups(self):- """Mixed MXFP4+MXFP8: override layers get explicit targets; default gets ["Linear"]."""- default = {"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"}- layer_config = {- "model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 32, "data_type": "mx_fp"},- "model.layers.0.self_attn.k_proj": {"bits": 8, "group_size": 32, "data_type": "mx_fp"},- }- quantized = [- "model.layers.0.self_attn.q_proj",- "model.layers.0.self_attn.k_proj",- "model.layers.0.mlp.fc1",- "model.layers.0.mlp.fc2",- ]- ignored = ["lm_head"]- cfg = _build_mxfp_quantization_config(default, quantized, ignored, layer_config=layer_config)-- assert cfg["format"] == "mixed-precision"- assert len(cfg["config_groups"]) == 2-- # MXFP8 group (override, should come first) — explicit targets- mxfp8_group = next(g for g in cfg["config_groups"].values() if g["format"] == "mxfp8-quantized")- assert set(mxfp8_group["targets"]) == {- "model.layers.0.self_attn.q_proj",- "model.layers.0.self_attn.k_proj",- }-- # MXFP4 group (default) — catch-all- mxfp4_group = next(g for g in cfg["config_groups"].values() if g["format"] == "mxfp4-pack-quantized")- assert mxfp4_group["targets"] == ["Linear"]+ # Unit tests for _build_mxfp_quantization_config moved to+ # test/unit/test_cpu/utils/test_model_free_utils.py -> TestBuildMxfpConfig@require_compressed_tensorsdef test_e2e_mxfp_mixed(self, tmp_path):@@ -1015,44 +785,8 @@ def _make_deepseek_v4_mxfp8(out_f, in_f, block_h, block_w):return weight_fp8, scale-class TestExpandE8M0BlockScale:- """The coarse-block → per-group E8M0 scale expansion helper."""-- def test_expand_repeat_interleave(self):- # Coarse [2, 2] block scale for a [64, 128] weight, group_size=32.- scale = torch.tensor([[100, 101], [102, 103]], dtype=torch.uint8)- out = _expand_e8m0_block_scale(scale, out_features=64, in_features=128, group_size=32)- assert out.shape == (64, 4) # 128 // 32 == 4 groups- assert out.dtype == torch.uint8- # Row 0 (first 32 rows) maps to coarse row 0; cols 0..1 → coarse col 0, 2..3 → col 1.- assert out[0].tolist() == [100, 100, 101, 101]- assert out[63].tolist() == [102, 102, 103, 103]-- def test_expand_noop_when_already_fine(self):- scale = torch.full((64, 4), 127, dtype=torch.uint8)- out = _expand_e8m0_block_scale(scale, out_features=64, in_features=128, group_size=32)- assert out.shape == (64, 4) and torch.equal(out, scale)-- def test_expand_invalid_shape_raises(self):- scale = torch.full((3, 4), 127, dtype=torch.uint8)- with pytest.raises(ValueError):- _expand_e8m0_block_scale(scale, out_features=64, in_features=128, group_size=32)---class TestDeepseekV4MXFP8Source:- """deepseek_v4 source models stored as float8 weights + coarse E8M0 scales."""-- def test_resolve_model_type(self):- core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")- core.config = _DEEPSEEK_V4_CFG- core._resolve_model_type()- assert core.model_type == "deepseek_v4"-- def test_resolve_model_type_negative(self):- core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")- core.config = _LLAMA_CFG- core._resolve_model_type()- assert core.model_type == "llama"+# TestExpandE8M0BlockScale moved to test/unit/test_cpu/utils/test_model_free_utils.py+# TestDeepseekV4MXFP8Source (resolve_model_type tests) moved to test/unit/test_cpu/utils/test_model_free_utils.py# ===========================================================================@@ -1072,117 +806,12 @@ _LLMCOMPRESSOR_MIXED_CFG = {class TestLLMCompressorMXFPSource:- """llm-compressor MXFP8/MXFP4 source models (generic non-deepseek_v4 path)."""-- def test_resolve_model_type_qwen3(self):- core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")- core.config = _LLMCOMPRESSOR_MXFP_CFG_FP8- core._resolve_model_type()- assert core.model_type == "qwen3"-- def test_resolve_model_type_mixed(self):- core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")- core.config = _LLMCOMPRESSOR_MIXED_CFG- core._resolve_model_type()- assert core.model_type == "qwen3"-- def test_resolve_model_type_negative_not_compressed_tensors(self):- core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")- core.config = _LLAMA_CFG- core._resolve_model_type()- assert core.model_type == "llama"-- def test_passthrough_mxfp8_same_target(self):- """MXFP8 source + MXFP8 target → passthrough (bytes preserved)."""- weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)- weight_scale = torch.full((64, 4), 127, dtype=torch.uint8)- raw = {- "layer.weight": weight_fp8.clone(),- "layer.weight_scale": weight_scale.clone(),- }- matcher = _matcher(default={"bits": 8, "group_size": 32, "sym": True, "data_type": "mx_fp"})- raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)-- assert layers == ["layer"]- assert passthrough["layer.weight"].dtype == torch.float8_e4m3fn- assert torch.equal(passthrough["layer.weight"].view(torch.uint8), weight_fp8.view(torch.uint8))- assert passthrough["layer.weight_scale"].dtype == torch.uint8- assert "layer.weight" not in raw_out- assert "layer.weight_scale" not in raw_out-- def test_passthrough_mxfp4_same_target(self):- """MXFP4 source + MXFP4 target → passthrough."""- weight_packed = torch.randint(0, 255, (64, 64), dtype=torch.uint8)- weight_scale = torch.full((64, 4), 127, dtype=torch.uint8)- raw = {- "layer.weight_packed": weight_packed.clone(),- "layer.weight_scale": weight_scale.clone(),- }- matcher = _matcher(default={"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"})- raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)-- assert layers == ["layer"]- assert torch.equal(passthrough["layer.weight_packed"], weight_packed)- assert passthrough["layer.weight_scale"].dtype == torch.uint8- assert "layer.weight_packed" not in raw_out-- def test_mxfp8_dequant_when_int_target(self):- """MXFP8 source + int target → dequantized to bf16 .weight."""- weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)- weight_scale = torch.full((64, 4), 127, dtype=torch.uint8) # scale 1.0- raw = {- "layer.weight": weight_fp8.clone(),- "layer.weight_scale": weight_scale.clone(),- }- matcher = _matcher(default={"bits": 4, "group_size": 128, "sym": True, "data_type": "int"})- raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)-- assert layers == [] and passthrough == {}- assert raw_out["layer.weight"].dtype == torch.bfloat16- assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.bfloat16))- assert "layer.weight_scale" not in raw_out-- def test_mixed_passthrough_and_dequant(self):- """Mixed: MXFP8 layer passthrough + MXFP4 layer dequanted (target MXFP8)."""- weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)- scale_fp8 = torch.full((64, 4), 127, dtype=torch.uint8)- weight_packed = torch.randint(0, 255, (64, 64), dtype=torch.uint8)- scale_packed = torch.full((64, 4), 127, dtype=torch.uint8)- raw = {- "attn.weight": weight_fp8.clone(),- "attn.weight_scale": scale_fp8.clone(),- "mlp.weight_packed": weight_packed.clone(),- "mlp.weight_scale": scale_packed.clone(),- }- matcher = _matcher(default={"bits": 8, "group_size": 32, "sym": True, "data_type": "mx_fp"})- raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)-- assert "attn" in layers # fp8 layer passthrough- assert passthrough["attn.weight"].dtype == torch.float8_e4m3fn- # mlp fp4 layer dequantized because target is MXFP8 (bits=8 != 4)- assert "mlp" not in layers- assert raw_out["mlp.weight"].dtype == torch.bfloat16- assert raw_out["mlp.weight"].shape == (64, 128)-- def test_noop_without_mxfp_tensors(self):- """No MXFP tensors → input returned unchanged."""- raw = {"layer.weight": torch.randn(64, 128, dtype=torch.bfloat16)}- matcher = _matcher(default={"bits": 8, "group_size": 32, "data_type": "mx_fp"})- raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)- assert raw_out is raw and passthrough == {} and layers == []-- def test_reject_non_mxfp_packed_float_scale(self):- """weight_packed + float scale must NOT be treated as MXFP source."""- raw = {- "layer.weight_packed": torch.randint(0, 255, (64, 64), dtype=torch.uint8),- "layer.weight_scale": torch.ones(64, 8, dtype=torch.float16),- }- matcher = _matcher(default={"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"})- raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)+ """llm-compressor MXFP8/MXFP4 source models (generic non-deepseek_v4 path).- assert raw_out is raw- assert passthrough == {}- assert layers == []+ Unit tests for _handle_mxfp_source_tensors and _dequant_mxfp_tensors moved to+ test/unit/test_cpu/utils/test_model_free_utils.py -> TestHandleMXFPSourceTensors.+ resolve_model_type tests moved to test/unit/test_cpu/utils/test_model_free_utils.py.+ """@require_compressed_tensorsdef test_e2e_mxfp8_passthrough(self, tmp_path):@@ -1216,57 +845,8 @@ class TestLLMCompressorMXFPSource:assert torch.equal(wp.view(torch.uint8), weight_fp8.view(torch.uint8))assert ws.dtype == torch.uint8- def test_convert_passthrough_when_target_mxfp8(self):- """Target MXFP8 → converted tensors emitted directly (weight bytes preserved)."""- weight_fp8, scale = _make_deepseek_v4_mxfp8(64, 128, block_h=32, block_w=64)- raw = {"layer.weight": weight_fp8.clone(), "layer.scale": scale.clone()}- matcher = _matcher(default={"bits": 8, "group_size": 32, "sym": True, "data_type": "mx_fp"})- raw_out, state = _preprocess_model_type_source_tensors(raw, model_type="deepseek_v4")- raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw_out, matcher, source_state=state)-- assert layers == ["layer"]- # weight kept as float8 under .weight; scale expanded to per-group uint8.- assert passthrough["layer.weight"].dtype == torch.float8_e4m3fn- assert torch.equal(passthrough["layer.weight"].view(torch.uint8), weight_fp8.view(torch.uint8))- assert passthrough["layer.weight_scale"].dtype == torch.uint8- assert passthrough["layer.weight_scale"].shape == (64, 4)- # source tensors consumed, nothing left in raw.- assert "layer.weight" not in raw_out- assert "layer.scale" not in raw_out-- def test_convert_dequant_when_target_int(self):- """Non-MXFP8 target → tensors dequantized to bfloat16 .weight."""- weight_fp8, scale = _make_deepseek_v4_mxfp8(64, 128, block_h=32, block_w=64)- raw = {"layer.weight": weight_fp8.clone(), "layer.scale": scale.clone()}- matcher = _matcher(default={"bits": 4, "group_size": 128, "sym": True, "data_type": "int"})- raw_out, state = _preprocess_model_type_source_tensors(raw, model_type="deepseek_v4")- raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw_out, matcher, source_state=state)-- assert layers == [] and passthrough == {}- assert raw_out["layer.weight"].dtype == torch.bfloat16- assert raw_out["layer.weight"].shape == (64, 128)- # scale 1.0 → dequant weight equals fp8 cast to bf16.- assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.bfloat16))- assert "layer.scale" not in raw_out and "layer.weight_scale" not in raw_out-- def test_convert_noop_without_quantized(self):- """No float8/packed weights → input returned unchanged."""- raw = {"layer.weight": torch.randn(64, 128, dtype=torch.bfloat16)}- matcher = _matcher(default={"bits": 8, "group_size": 32, "data_type": "mx_fp"})- raw_out, state = _preprocess_model_type_source_tensors(raw, model_type="deepseek_v4")- assert state == {}- raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw_out, matcher, source_state=state)- assert raw_out is raw and passthrough == {} and layers == []-- def test_dequant_mxfp_tensors_mxfp8(self):- """Generic MXFP dequant: float8 .weight + .weight_scale → bf16."""- weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)- weight_scale = torch.full((64, 4), 127, dtype=torch.uint8) # scale 1.0- raw = {"layer.weight": weight_fp8.clone(), "layer.weight_scale": weight_scale.clone()}- out = _dequant_mxfp_tensors(raw)- assert out["layer.weight"].dtype == torch.bfloat16- assert torch.allclose(out["layer.weight"], weight_fp8.to(torch.bfloat16))- assert "layer.weight_scale" not in out+ # Unit tests test_convert_* and test_dequant_* moved to+ # test/unit/test_cpu/utils/test_model_free_utils.py -> TestHandleMXFPSourceTensors@require_compressed_tensorsdef test_e2e_passthrough_mxfp8_target(self, tmp_path):@@ -1393,74 +973,10 @@ class TestSchemeValidation:# ===========================================================================-# kimi_k25 INT4 packed source models+# kimi_k25 INT4 packed source models — moved to test/unit/test_cpu/utils/test_model_free_utils.py# ===========================================================================-_KIMI_K25_CFG = {- "architectures": ["KimiK25ForConditionalGeneration"],- "model_type": "kimi_k25",- "quantization_config": {- "quant_method": "compressed-tensors",- "weights": {"num_bits": 4, "type": "int", "group_size": 8, "symmetric": True},- },-}---class TestKimiK25Int4Source:- def test_kimi_k25_int4_dequant_helper(self):- raw = {- "layer.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),- "layer.weight_scale": torch.ones(128, 16, dtype=torch.float16),- }- out = _handle_model_type_low_precision_source_tensors(- raw,- model_type="kimi_k25",- source_quant_config=_KIMI_K25_CFG["quantization_config"],- device="cpu",- )- assert "layer.weight" in out- assert out["layer.weight"].dtype == torch.bfloat16- assert out["layer.weight"].shape == (128, 128)- assert "layer.weight_packed" not in out- assert "layer.weight_scale" not in out-- @require_compressed_tensors- def test_kimi_k25_int4_to_mxfp4_via_model_free(self, tmp_path):- tensors = {- "model.layers.0.mlp.fc1.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),- "model.layers.0.mlp.fc1.weight_scale": torch.ones(128, 16, dtype=torch.float16),- "lm_head.weight": torch.randn(1000, 128),- }- model_dir = _make_model_dir(tmp_path, _KIMI_K25_CFG, tensors)- output_dir = str(tmp_path / "output")-- _ModelFreeCompressorCore(- model_name_or_path=model_dir,- output_dir=output_dir,- scheme="MXFP4",- format="llm_compressor",- ).run()-- qc = _read_qconfig(output_dir)- assert qc["format"] == "mxfp4-pack-quantized"- assert qc["quant_method"] == "compressed-tensors"-- found_scale_dtype = None- found_packed_shape = None- for fname in os.listdir(output_dir):- if not fname.endswith(".safetensors"):- continue- with safe_open(os.path.join(output_dir, fname), framework="pt") as sf:- if "model.layers.0.mlp.fc1.weight_scale" in sf.keys():- found_scale_dtype = sf.get_tensor("model.layers.0.mlp.fc1.weight_scale").dtype- found_packed_shape = sf.get_tensor("model.layers.0.mlp.fc1.weight_packed").shape-- # Re-quantized MXFP4 scales are uint8 E8M0 (source INT4 scales were fp16).- assert found_scale_dtype == torch.uint8- assert found_packed_shape == (128, 64)--# ===========================================================================# CLI auto-routing# ===========================================================================@@ -1516,24 +1032,7 @@ class TestCliAutoRouting:assert auto_route is False-# ===========================================================================-# Predefined ignore layers-# ===========================================================================---class TestPredefinedIgnoreLayers:- def test_normal_model_empty(self):- assert get_predefined_ignore_layers_from_config({"architectures": ["LlamaForCausalLM"]}) == []-- def test_step3p5_ignore_layers(self):- cfg = {"model_type": "step3p5"}- assert get_predefined_ignore_layers_from_config(cfg) == [- "g_proj",- "moe.gate",- "eh_proj",- "shared_head",- "layers.45",- ]+# TestPredefinedIgnoreLayers moved to test/unit/test_cpu/utils/test_model_free_utils.py# ===========================================================================@@ -1610,7 +1109,7 @@ _TRANSFORMER_TENSORS = {}-class TestCopyMetadataSubfolders:+class TestKimiK25Int4Source:"""Tests for _copy_metadata_files including subdirectory handling."""def test_non_diffusion_copies_subfolders(self, tmp_path):@@ -1805,80 +1304,12 @@ class TestResolveShardParallelism:class TestModelFreeAutoScheme:- """Model-free support for ``AutoScheme`` mixed-bit selection."""-- def test_looks_like_auto_scheme(self):- from auto_round import AutoScheme-- assert _looks_like_auto_scheme(AutoScheme(avg_bits=3, options=("W2A16", "W4A16")))- assert not _looks_like_auto_scheme("W4A16")- assert not _looks_like_auto_scheme(QuantizationScheme(bits=4))-- def test_validate_options_int_family(self):- from auto_round import AutoScheme-- assert _validate_auto_scheme_options(AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "W8A16"))) == "int"-- def test_validate_options_int_family_with_bf16(self):- from auto_round import AutoScheme-- assert _validate_auto_scheme_options(AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16"))) == "int"+ """Model-free support for ``AutoScheme`` mixed-bit selection.- def test_validate_options_mxfp_family(self):- from auto_round import AutoScheme-- assert _validate_auto_scheme_options(AutoScheme(avg_bits=6, options=("MXFP4", "MXFP8"))) == "mx_fp"-- def test_validate_options_mixed_family_raises(self):- from auto_round import AutoScheme-- with pytest.raises(ValueError, match="mix INT and MXFP"):- _validate_auto_scheme_options(AutoScheme(avg_bits=4, options=("W4A16", "MXFP4")))-- @pytest.mark.parametrize(- "options",- [- ("W3A16", "W4A16"),- ("GGUF:Q4_K_M", "W8A16"),- ("NVFP4", "W4A16"),- ("MXFP4_RCEIL", "MXFP4"),- ],- )- def test_validate_options_unsupported_raises(self, options):- from auto_round import AutoScheme-- with pytest.raises(ValueError, match="unsupported option"):- _validate_auto_scheme_options(AutoScheme(avg_bits=4, options=options))-- def test_convert_layer_config(self):- generated = {- "model.layers.0.q_proj": {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"},- "model.layers.0.k_proj": {"bits": 2, "group_size": 128, "sym": True, "data_type": "int"},- "model.layers.0.v_proj": {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"},- "model.embed_tokens": {"bits": 16, "group_size": 128, "sym": True, "data_type": "int"},- }- base_scheme, per_layer, fp16_layers = _convert_auto_scheme_layer_config(generated)- # Most common quantized scheme (4-bit) becomes the base.- assert base_scheme.bits == 4 and base_scheme.group_size == 128- assert per_layer["model.layers.0.k_proj"]["bits"] == 2- assert "model.embed_tokens" not in per_layer- assert fp16_layers == ["model.embed_tokens"]-- def test_convert_layer_config_infers_mxfp_bits_from_dtype_alias(self):- generated = {- "model.layers.0.q_proj": {"group_size": 32, "sym": True, "data_type": "mxfp8"},- "model.layers.0.k_proj": {"group_size": 32, "sym": True, "data_type": "MXFP4"},- }-- base_scheme, per_layer, fp16_layers = _convert_auto_scheme_layer_config(generated)-- assert fp16_layers == []- assert per_layer["model.layers.0.q_proj"]["bits"] == 8- assert per_layer["model.layers.0.q_proj"]["data_type"] == "mx_fp"- assert per_layer["model.layers.0.k_proj"]["bits"] == 4- assert per_layer["model.layers.0.k_proj"]["data_type"] == "mx_fp"- assert base_scheme.data_type == "mx_fp"- assert base_scheme.bits in (4, 8)+ Pure unit tests for helper functions (_looks_like_auto_scheme,+ _validate_auto_scheme_options, _convert_auto_scheme_layer_config) have been+ moved to test/unit/test_cpu/utils/test_model_free_utils.py -> TestAutoSchemeHelpers.+ """@pytest.mark.timeout(120)def test_e2e_int_auto_scheme(self, tmp_path, tiny_opt_model_path):@@ -1921,197 +1352,11 @@ class TestMXFPAutoRoundFormat:The weight tensors on disk are identical to the llm_compressor path;only the ``quantization_config`` metadata differs.- """-- # ------------------------------------------------------------------- # Unit tests: _build_mxfp_autoround_quantization_config- # -------------------------------------------------------------------- @require_compressed_tensors- def test_mxfp4_top_level_fields(self):- """MXFP4 scheme → all mandatory top-level fields are correct."""- from dataclasses import asdict-- from auto_round.schemes import PRESET_SCHEMES-- default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP4"]).items() if v is not None}- cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])- assert cfg["quant_method"] == "auto-round"- assert cfg["packing_format"] == "auto_round:llm_compressor"- assert cfg["bits"] == 4- assert cfg["group_size"] == 32- assert cfg["data_type"] == "mx_fp"- assert cfg["sym"] is True- assert cfg["enable_quanted_input"] is False- assert cfg["model_free"] is True- assert "autoround_version" in cfg-- @require_compressed_tensors- def test_mxfp4_activation_fields(self):- """MXFP4 scheme carries act_* fields matching the real AutoRound output."""- from dataclasses import asdict-- from auto_round.schemes import PRESET_SCHEMES-- default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP4"]).items() if v is not None}- cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])- assert cfg["act_bits"] == 4- assert cfg["act_data_type"] == "mx_fp"- assert cfg["act_dynamic"] is True- assert cfg["act_group_size"] == 32- assert cfg["act_sym"] is True- @require_compressed_tensors- def test_mxfp8_bits(self):- """MXFP8 scheme → bits=8, act_bits=8."""- from dataclasses import asdict-- from auto_round.schemes import PRESET_SCHEMES-- default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP8"]).items() if v is not None}- cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])- assert cfg["bits"] == 8- assert cfg["act_bits"] == 8- assert cfg["packing_format"] == "auto_round:llm_compressor"-- @require_compressed_tensors- def test_ignored_layers_in_extra_config(self):- """Ignored layers (non-embed/conv) go into extra_config as bits=16."""- default = {- "bits": 4,- "group_size": 32,- "sym": True,- "data_type": "mx_fp",- "act_bits": 4,- "act_data_type": "mx_fp",- "act_dynamic": True,- "act_group_size": 32,- "act_sym": True,- }- ignored = ["lm_head", "model.embed_tokens", "model.conv1", "model.layers.0.mlp.gate"]- cfg = _build_mxfp_autoround_quantization_config(- default,- quantized_layers=["model.layers.0.fc1"],- ignored_layers=ignored,- )- extra = cfg.get("extra_config", {})- full_precision = {"bits": 16, "data_type": "float", "act_bits": 16, "act_data_type": "float"}- assert extra.get("lm_head") == full_precision- assert extra.get("model.layers.0.mlp.gate") == full_precision- # embed / conv are non-Linear — filtered out- assert "model.embed_tokens" not in extra- assert "model.conv1" not in extra-- @require_compressed_tensors- def test_ignored_lm_head_loads_as_full_precision(self):- """An ignored lm_head must not inherit MXFP activation quantization."""- from types import SimpleNamespace-- from transformers import OPTConfig, OPTForCausalLM-- from auto_round.inference.convert_model import get_layer_config- from auto_round.utils import check_to_quantized-- default = {- "bits": 4,- "group_size": 32,- "sym": True,- "data_type": "mx_fp",- "act_bits": 4,- "act_data_type": "mx_fp",- "act_dynamic": True,- "act_group_size": 32,- "act_sym": True,- }- quantization_config = _build_mxfp_autoround_quantization_config(- default,- quantized_layers=["model.decoder.layers.0.fc1"],- ignored_layers=["lm_head"],- block_name_to_quantize="model.decoder.layers",- )- model = OPTForCausalLM(- OPTConfig(- hidden_size=32,- ffn_dim=64,- num_hidden_layers=1,- num_attention_heads=4,- vocab_size=64,- word_embed_proj_dim=32,- )- )-- layer_configs = get_layer_config(model, SimpleNamespace(**quantization_config))- lm_head_config = layer_configs["lm_head"]-- assert lm_head_config.bits == 16- assert lm_head_config.act_bits == 16- assert lm_head_config.data_type == "float"- assert lm_head_config.act_data_type == "float"- assert not check_to_quantized(lm_head_config)-- @require_compressed_tensors- def test_quantized_lm_head_in_extra_config(self):- """When lm_head is quantized it must appear in extra_config with its scheme."""- default = {- "bits": 4,- "group_size": 32,- "sym": True,- "data_type": "mx_fp",- "act_bits": 4,- "act_data_type": "mx_fp",- "act_dynamic": True,- "act_group_size": 32,- "act_sym": True,- }- cfg = _build_mxfp_autoround_quantization_config(- default,- quantized_layers=["model.layers.0.fc1", "lm_head"],- ignored_layers=[],- )- extra = cfg.get("extra_config", {})- assert "lm_head" in extra- assert extra["lm_head"]["bits"] == 4- assert extra["lm_head"]["data_type"] == "mx_fp"-- @require_compressed_tensors- def test_build_quantization_config_routes_mxfp_to_autoround(self):- """_build_quantization_config with format='auto_round' routes MXFP to auto-round style."""- default = {- "bits": 4,- "group_size": 32,- "sym": True,- "data_type": "mx_fp",- "act_bits": 4,- "act_data_type": "mx_fp",- "act_dynamic": True,- "act_group_size": 32,- "act_sym": True,- }- cfg = _build_quantization_config(- default_scheme=default,- layer_config={},- ignore_patterns=[],- quantized_layers=["layer.fc1"],- ignored_layers=[],- format="auto_round",- )- assert cfg["quant_method"] == "auto-round"- assert cfg["packing_format"] == "auto_round:llm_compressor"-- @require_compressed_tensors- def test_build_quantization_config_mxfp_llmcompressor_path_unchanged(self):- """_build_quantization_config with format='llm_compressor' still produces compressed-tensors."""- default = {"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"}- cfg = _build_quantization_config(- default_scheme=default,- layer_config={},- ignore_patterns=[],- quantized_layers=["layer.fc1"],- ignored_layers=["lm_head"],- format="llm_compressor",- )- assert cfg["quant_method"] == "compressed-tensors"- assert cfg["format"] == "mxfp4-pack-quantized"+ Unit tests for _build_mxfp_autoround_quantization_config and+ _build_quantization_config routing have been moved to+ test/unit/test_cpu/utils/test_model_free_utils.py -> TestBuildMxfpAutoRoundConfig.+ """# ------------------------------------------------------------------# End-to-end tests via AutoRound.quantize_and_savediff --git a/test/unit/test_cpu/utils/test_model_free_cross_shard.py b/test/unit/test_cpu/utils/test_model_free_cross_shard.pydeleted file mode 100644index e2f0c4700..000000000--- a/test/unit/test_cpu/utils/test_model_free_cross_shard.py+++ /dev/null@@ -1,270 +0,0 @@-# Copyright (c) 2026 Intel Corporation-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.--"""Unit tests for cross-shard FP8 scale handling in model_free_utils."""--import json-import os-import tempfile--import pytest-import torch--from auto_round.utils.model_free_utils import (- _build_cross_shard_pairs_from_weight_map,- _hydrate_missing_fp8_scales_from_index,-)--# ----------------------------------------------------------------------------# _build_cross_shard_pairs_from_weight_map-# ------------------------------------------------------------------------------class TestBuildCrossShardPairs:- """Tests for _build_cross_shard_pairs_from_weight_map."""-- def _make_weight_map(self, entries: dict[str, str]) -> dict[str, str]:- return dict(entries)-- def test_no_fp8_entries_returns_empty(self):- weight_map = {- "model.layer.weight": "shard-00001.safetensors",- "model.layer.bias": "shard-00001.safetensors",- }- recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)- assert recipient_to_donors == {}- assert donor_shard_tensors == {}-- def test_same_shard_scale_not_cross(self):- """weight and weight_scale_inv in the same shard → not a cross-shard pair."""- weight_map = {- "model.layer.weight": "shard-00001.safetensors",- "model.layer.weight_scale_inv": "shard-00001.safetensors",- }- recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)- assert recipient_to_donors == {}- assert donor_shard_tensors == {}-- def test_cross_shard_single_pair(self):- """weight in shard-1, weight_scale_inv in shard-2."""- weight_map = {- "model.layer.weight": "shard-00001.safetensors",- "model.layer.weight_scale_inv": "shard-00002.safetensors",- }- recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)-- assert "shard-00001.safetensors" in recipient_to_donors- donor_map = recipient_to_donors["shard-00001.safetensors"]- assert "shard-00002.safetensors" in donor_map- assert "model.layer.weight_scale_inv" in donor_map["shard-00002.safetensors"]-- assert "shard-00002.safetensors" in donor_shard_tensors- assert "model.layer.weight_scale_inv" in donor_shard_tensors["shard-00002.safetensors"]-- def test_cross_shard_multiple_layers_one_donor(self):- """Multiple layers whose scale_inv all live in the same donor shard."""- weight_map = {- "model.layer0.weight": "shard-00001.safetensors",- "model.layer0.weight_scale_inv": "shard-00002.safetensors",- "model.layer1.weight": "shard-00001.safetensors",- "model.layer1.weight_scale_inv": "shard-00002.safetensors",- }- recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)-- donor_map = recipient_to_donors["shard-00001.safetensors"]- scales = donor_map["shard-00002.safetensors"]- assert "model.layer0.weight_scale_inv" in scales- assert "model.layer1.weight_scale_inv" in scales- assert len(donor_shard_tensors["shard-00002.safetensors"]) == 2-- def test_cross_shard_multiple_donors(self):- """Recipient shard needs scales from two different donor shards."""- weight_map = {- "model.layerA.weight": "shard-00001.safetensors",- "model.layerA.weight_scale_inv": "shard-00002.safetensors",- "model.layerB.weight": "shard-00001.safetensors",- "model.layerB.weight_scale_inv": "shard-00003.safetensors",- }- recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)-- donor_map = recipient_to_donors["shard-00001.safetensors"]- assert "shard-00002.safetensors" in donor_map- assert "shard-00003.safetensors" in donor_map- assert len(donor_shard_tensors) == 2-- def test_scale_inv_without_matching_weight_ignored(self):- """weight_scale_inv present in weight_map but no corresponding .weight → ignored."""- weight_map = {- "model.layer.weight_scale_inv": "shard-00002.safetensors",- # no "model.layer.weight" key at all- }- recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)- assert recipient_to_donors == {}- assert donor_shard_tensors == {}-- def test_empty_weight_map(self):- recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map({})- assert recipient_to_donors == {}- assert donor_shard_tensors == {}---# ----------------------------------------------------------------------------# _hydrate_missing_fp8_scales_from_index-# ------------------------------------------------------------------------------def _write_fake_fp8_shard(path: str, tensors: dict[str, torch.Tensor]) -> None:- """Save a dict of tensors as a safetensors file."""- from safetensors.torch import save_file-- save_file(tensors, path)---def _write_index_json(directory: str, weight_map: dict[str, str]) -> str:- index_path = os.path.join(directory, "model.safetensors.index.json")- with open(index_path, "w") as f:- json.dump({"weight_map": weight_map}, f)- return index_path---class TestHydrateMissingFp8Scales:- """Tests for _hydrate_missing_fp8_scales_from_index."""-- def test_non_safetensors_shard_returns_unchanged(self, tmp_path):- raw = {"w": torch.zeros(4)}- result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / "model.bin"))- assert result is raw-- def test_no_fp8_weights_returns_unchanged(self, tmp_path):- shard_path = str(tmp_path / "shard-00001.safetensors")- # BF16 weight — not FP8, so nothing to hydrate- raw = {"model.layer.weight": torch.zeros(4, dtype=torch.bfloat16)}- result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)- assert result is raw-- def test_all_scales_present_returns_unchanged(self, tmp_path):- shard_path = str(tmp_path / "shard-00001.safetensors")- raw = {- "model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),- "model.layer.weight_scale_inv": torch.ones(1),- }- result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)- assert "model.layer.weight_scale_inv" in result-- def test_cross_shard_hydration_local_mode(self, tmp_path):- """Recipient shard gets scale_inv from donor shard in local (non-streaming) mode."""- donor_name = "shard-00002.safetensors"- recipient_name = "shard-00001.safetensors"- scale_name = "model.layer.weight_scale_inv"-- # Write donor shard with scale_inv- donor_path = tmp_path / donor_name- _write_fake_fp8_shard(str(donor_path), {scale_name: torch.ones(1)})-- # Write index.json in the same directory- weight_map = {- "model.layer.weight": recipient_name,- scale_name: donor_name,- }- _write_index_json(str(tmp_path), weight_map)-- # Recipient shard has FP8 weight but no scale_inv- recipient_path = tmp_path / recipient_name- raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}-- result = _hydrate_missing_fp8_scales_from_index(raw, str(recipient_path))- assert scale_name in result, "scale_inv should be hydrated from donor shard"- assert result[scale_name].dtype == torch.float32 or result[scale_name].numel() == 1-- def test_cross_shard_hydration_streaming_mode(self, tmp_path):- """In streaming mode, index.json lives in work_dir, shards in cache subdir."""- work_dir = tmp_path / "work_dir"- cache_dir = work_dir / ".cache" / "model_free_source_shards"- work_dir.mkdir()- cache_dir.mkdir(parents=True)-- donor_name = "shard-00002.safetensors"- recipient_name = "shard-00001.safetensors"- scale_name = "model.layer.weight_scale_inv"-- # Donor shard lives in cache subdir- _write_fake_fp8_shard(str(cache_dir / donor_name), {scale_name: torch.ones(1)})-- # Index.json lives in work_dir (not in cache subdir)- weight_map = {- "model.layer.weight": recipient_name,- scale_name: donor_name,- }- _write_index_json(str(work_dir), weight_map)-- # Recipient shard path is in cache dir- recipient_path = cache_dir / recipient_name- raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}-- result = _hydrate_missing_fp8_scales_from_index(- raw,- str(recipient_path),- index_dir=str(work_dir),- donor_shard_dir=str(cache_dir),- )- assert scale_name in result, "streaming mode: scale_inv should be hydrated via index_dir/donor_shard_dir params"-- def test_missing_index_json_returns_unchanged(self, tmp_path):- """If no index.json exists, raw_tensors is returned as-is (no crash)."""- shard_path = str(tmp_path / "shard-00001.safetensors")- raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}- result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)- assert result is raw-- def test_donor_shard_missing_on_disk_returns_unchanged(self, tmp_path):- """Index references a donor shard that doesn't exist on disk → graceful skip."""- scale_name = "model.layer.weight_scale_inv"- recipient_name = "shard-00001.safetensors"- donor_name = "shard-00002.safetensors" # not written-- weight_map = {- "model.layer.weight": recipient_name,- scale_name: donor_name,- }- _write_index_json(str(tmp_path), weight_map)-- raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}- result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / recipient_name))- assert scale_name not in result # hydration skipped silently-- def test_multiple_layers_hydrated_from_single_donor(self, tmp_path):- """Multiple missing scale_inv tensors are hydrated in a single donor open."""- donor_name = "shard-00002.safetensors"- recipient_name = "shard-00001.safetensors"- scales = {- "model.layerA.weight_scale_inv": torch.ones(1),- "model.layerB.weight_scale_inv": torch.ones(1),- }- _write_fake_fp8_shard(str(tmp_path / donor_name), scales)-- weight_map = {- "model.layerA.weight": recipient_name,- "model.layerA.weight_scale_inv": donor_name,- "model.layerB.weight": recipient_name,- "model.layerB.weight_scale_inv": donor_name,- }- _write_index_json(str(tmp_path), weight_map)-- raw = {- "model.layerA.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),- "model.layerB.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),- }- result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / recipient_name))- assert "model.layerA.weight_scale_inv" in result- assert "model.layerB.weight_scale_inv" in resultdiff --git a/test/unit/test_cpu/utils/test_model_free_utils.py b/test/unit/test_cpu/utils/test_model_free_utils.pynew file mode 100644index 000000000..4c7568a31--- /dev/null+++ b/test/unit/test_cpu/utils/test_model_free_utils.py@@ -0,0 +1,1447 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Unit tests for model_free_utils helpers."""++import json+import os+import tempfile+from unittest.mock import Mock++import pytest+import torch+from safetensors import safe_open+from safetensors.torch import save_file++from auto_round.compressors.model_free import (+ _ModelFreeCompressorCore,+ get_predefined_ignore_layers_from_config,+)+from auto_round.schemes import QuantizationScheme+from auto_round.utils.model import is_model_free_route+from auto_round.utils.model_free_utils import (+ _add_routed_experts_if_moe,+ _build_cross_shard_pairs_from_weight_map,+ _build_mxfp_autoround_quantization_config,+ _build_mxfp_quantization_config,+ _build_quantization_config,+ _convert_auto_scheme_layer_config,+ _dequant_fp8_tensors,+ _dequant_mxfp_tensors,+ _expand_e8m0_block_scale,+ _handle_mxfp_source_tensors,+ _hydrate_missing_fp8_scales_from_index,+ _looks_like_auto_scheme,+ _PatternMatcher,+ _process_shard,+ _quantize_weight_mxfp,+ _validate_auto_scheme_options,+)+from auto_round.utils.model_free_utils import (+ handle_model_type_low_precision_source_tensors as _handle_model_type_low_precision_source_tensors,+)+from auto_round.utils.model_free_utils import (+ preprocess_model_type_source_tensors as _preprocess_model_type_source_tensors,+)++from ...envs import require_compressed_tensors+++def _make_model_dir(tmp_path, config, tensors, *, multi_shard=False):+ """Create a minimal local model directory with config.json and safetensors."""+ model_dir = str(tmp_path / "source_model")+ os.makedirs(model_dir, exist_ok=True)+ with open(os.path.join(model_dir, "config.json"), "w") as f:+ json.dump(config, f)++ if not multi_shard:+ save_file(tensors, os.path.join(model_dir, "model.safetensors"))+ else:+ keys = list(tensors.keys())+ mid = max(1, len(keys) // 2)+ shard1 = {k: tensors[k] for k in keys[:mid]}+ shard2 = {k: tensors[k] for k in keys[mid:]}+ save_file(shard1, os.path.join(model_dir, "model-00001-of-00002.safetensors"))+ save_file(shard2, os.path.join(model_dir, "model-00002-of-00002.safetensors"))+ weight_map = {}+ for k in keys[:mid]:+ weight_map[k] = "model-00001-of-00002.safetensors"+ for k in keys[mid:]:+ weight_map[k] = "model-00002-of-00002.safetensors"+ with open(os.path.join(model_dir, "model.safetensors.index.json"), "w") as f:+ json.dump({"metadata": {"total_size": 0}, "weight_map": weight_map}, f)+ return model_dir+++def _read_qconfig(output_dir):+ with open(os.path.join(output_dir, "config.json")) as f:+ return json.load(f).get("quantization_config", {})+++def _read_output_keys(output_dir):+ keys = set()+ for f in os.listdir(output_dir):+ if f.endswith(".safetensors"):+ with safe_open(os.path.join(output_dir, f), framework="pt") as sf:+ keys.update(sf.keys())+ return keys+++# ---------------------------------------------------------------------------+# Shared helpers+# ---------------------------------------------------------------------------+++def _matcher(ignore=None, layer_config=None, default=None):+ return _PatternMatcher(ignore or [], layer_config or {}, default or {})+++# ---------------------------------------------------------------------------+# _add_routed_experts_if_moe+# TODO: Remove when the issue is fixed+# https://github.com/vllm-project/llm-compressor/issues/3069+# ---------------------------------------------------------------------------+++class TestAddRoutedExpertsIfMoe:+ """Unit tests for the _add_routed_experts_if_moe helper.++ TODO: Remove when the issue is fixed+ https://github.com/vllm-project/llm-compressor/issues/3069+ """++ def test_w1_pattern_triggers(self):+ targets = ["Linear"]+ layers = ["model.layers.0.mlp.experts.0.w1.weight"]+ result = _add_routed_experts_if_moe(targets, layers)+ assert "RoutedExperts" in result++ def test_w3_pattern_triggers(self):+ targets = ["Linear"]+ layers = ["model.layers.0.mlp.experts.0.w3.weight"]+ result = _add_routed_experts_if_moe(targets, layers)+ assert "RoutedExperts" in result++ def test_block_sparse_moe_pattern_triggers(self):+ targets = ["Linear"]+ layers = ["model.layers.0.block_sparse_moe.experts.0.w1.weight"]+ result = _add_routed_experts_if_moe(targets, layers)+ assert "RoutedExperts" in result++ def test_non_moe_no_change(self):+ targets = ["Linear"]+ layers = ["model.layers.0.mlp.down_proj.weight", "model.layers.0.mlp.up_proj.weight"]+ result = _add_routed_experts_if_moe(targets, layers)+ assert result == ["Linear"]+ assert "RoutedExperts" not in result++ def test_w2_alone_does_not_trigger(self):+ """w2 (down-projection) alone should not trigger RoutedExperts."""+ targets = ["Linear"]+ layers = ["model.layers.0.mlp.experts.0.w2.weight"]+ result = _add_routed_experts_if_moe(targets, layers)+ assert "RoutedExperts" not in result++ def test_already_present_not_duplicated(self):+ targets = ["Linear", "RoutedExperts"]+ layers = ["model.layers.0.mlp.experts.0.w1.weight"]+ result = _add_routed_experts_if_moe(targets, layers)+ assert result.count("RoutedExperts") == 1++ def test_original_list_not_mutated(self):+ targets = ["Linear"]+ layers = ["model.layers.0.mlp.experts.0.w1.weight"]+ _add_routed_experts_if_moe(targets, layers)+ assert targets == ["Linear"]++ def test_mixed_moe_and_non_moe_layers_triggers(self):+ """Even if only some layers are MoE, RoutedExperts should be injected."""+ targets = ["Linear"]+ layers = [+ "model.layers.0.mlp.down_proj.weight",+ "model.layers.1.mlp.experts.0.w1.weight",+ ]+ result = _add_routed_experts_if_moe(targets, layers)+ assert "RoutedExperts" in result++ def test_empty_layers_no_change(self):+ targets = ["Linear"]+ result = _add_routed_experts_if_moe(targets, [])+ assert result == ["Linear"]+++# ---------------------------------------------------------------------------+# _build_mxfp_quantization_config: RoutedExperts in single-scheme groups+# TODO: Remove when the issue is fixed+# https://github.com/vllm-project/llm-compressor/issues/3069+# ---------------------------------------------------------------------------++# MoE-like layer names (w1/w3)+_MOE_W1_W3_LAYERS = [+ "model.layers.0.mlp.experts.0.w1",+ "model.layers.0.mlp.experts.0.w2",+ "model.layers.0.mlp.experts.0.w3",+ "model.layers.1.mlp.experts.0.w1",+ "model.layers.1.mlp.experts.0.w2",+ "model.layers.1.mlp.experts.0.w3",+]++_DENSE_LAYERS = [+ "model.layers.0.mlp.down_proj",+ "model.layers.0.mlp.up_proj",+ "model.layers.0.mlp.gate_proj",+]+++def _get_targets_from_qconfig(qconfig: dict, group: str = "group_0") -> list:+ return qconfig.get("config_groups", {}).get(group, {}).get("targets", [])+++ "scheme_name,bits",+ [+ ("MXFP4", 4),+ ("MXFP8", 8),+ ],+)+class TestBuildMxfpQuantizationConfigSingleSchemeMoe:+ """Test RoutedExperts injection for single-group MXFP configs.++ TODO: Remove when the issue is fixed+ https://github.com/vllm-project/llm-compressor/issues/3069+ """++ def _default_scheme(self, bits):+ return {"bits": bits, "data_type": "mx_fp", "group_size": 32}++ def test_moe_layers_add_routed_experts(self, scheme_name, bits):+ """BF16-default with only MoE layers quantized via layer_config should+ include RoutedExperts in the single config group's targets."""+ # Use block_sparse_moe-style names so the layer_config pattern can+ # actually be matched by _PatternMatcher (layer ends with .w1 so+ # .w1. does NOT match as substring; block_sparse_moe does).+ moe_quantized = [+ "model.layers.0.block_sparse_moe.experts.0.w1",+ "model.layers.0.block_sparse_moe.experts.0.w3",+ ]+ default_scheme = {"bits": 16, "data_type": "float", "group_size": -1}+ layer_cfg = {".block_sparse_moe.": {"bits": bits, "data_type": "mx_fp", "group_size": 32}}+ qconfig = _build_mxfp_quantization_config(+ default_scheme=default_scheme,+ quantized_layers=moe_quantized,+ ignored_layers=[],+ layer_config=layer_cfg,+ )+ targets = _get_targets_from_qconfig(qconfig, "group_0")+ assert "RoutedExperts" in targets, f"Expected RoutedExperts in targets, got: {targets}"++ def test_dense_layers_no_routed_experts(self, scheme_name, bits):+ """Dense-only quantized layers should NOT get RoutedExperts."""+ # Use the exact layer name as key so PatternMatcher can match it+ # (suffix patterns like .down_proj. don't match layer names that end+ # with down_proj since the compiled regex requires a trailing dot).+ dense_layer = "model.layers.0.mlp.down_proj"+ default_scheme = {"bits": 16, "data_type": "float", "group_size": -1}+ layer_cfg = {dense_layer: {"bits": bits, "data_type": "mx_fp", "group_size": 32}}+ qconfig = _build_mxfp_quantization_config(+ default_scheme=default_scheme,+ quantized_layers=[dense_layer],+ ignored_layers=[],+ layer_config=layer_cfg,+ )+ targets = _get_targets_from_qconfig(qconfig, "group_0")+ assert "RoutedExperts" not in targets, f"Unexpected RoutedExperts in targets: {targets}"++ def test_default_mxfp_global_targets_not_affected(self, scheme_name, bits):+ """When the default scheme is already MXFP (not BF16), targets stay as+ ``['Linear']`` for a uniform-scheme model without layer_config."""+ default_scheme = {"bits": bits, "data_type": "mx_fp", "group_size": 32}+ qconfig = _build_mxfp_quantization_config(+ default_scheme=default_scheme,+ quantized_layers=_MOE_W1_W3_LAYERS,+ ignored_layers=[],+ layer_config=None,+ )+ targets = _get_targets_from_qconfig(qconfig, "group_0")+ # Uniform-scheme → targets should be the catch-all ["Linear"], no explicit layer list+ assert targets == ["Linear"]+++# ---------------------------------------------------------------------------+# _build_mxfp_quantization_config: RoutedExperts in mixed-precision groups+# TODO: Remove when the issue is fixed+# https://github.com/vllm-project/llm-compressor/issues/3069+# ---------------------------------------------------------------------------+++class TestBuildMxfpQuantizationConfigMixedPrecisionMoe:+ """RoutedExperts must be injected per config_group in mixed-precision configs.++ TODO: Remove when the issue is fixed+ https://github.com/vllm-project/llm-compressor/issues/3069+ """++ def test_mixed_moe_groups_both_get_routed_experts(self):+ """Mixed precision: MXFP4 default + MXFP8 dense override.+ The MXFP8 override group has an explicit dense-layer list;+ it must NOT contain RoutedExperts.+ The MXFP4 default group uses ["Linear"] — RoutedExperts not needed."""+ moe_layers = [+ "model.layers.0.block_sparse_moe.experts.0.w1",+ "model.layers.0.block_sparse_moe.experts.0.w3",+ ]+ dense_layer = "model.layers.0.mlp.down_proj"+ all_layers = moe_layers + [dense_layer]++ default_scheme = {"bits": 4, "data_type": "mx_fp", "group_size": 32}+ # Override down_proj to MXFP8 so it lands in an override group.+ layer_cfg = {dense_layer: {"bits": 8, "data_type": "mx_fp", "group_size": 32}}+ qconfig = _build_mxfp_quantization_config(+ default_scheme=default_scheme,+ quantized_layers=all_layers,+ ignored_layers=[],+ layer_config=layer_cfg,+ )+ assert qconfig.get("format") == "mixed-precision"+ config_groups = qconfig.get("config_groups", {})+ # The override group (MXFP8) has only dense_layer — no RoutedExperts.+ for grp in config_groups.values():+ t = grp.get("targets", [])+ if t != ["Linear"] and dense_layer in str(t):+ assert "RoutedExperts" not in t, f"Dense override group should not have RoutedExperts: {t}"++ def test_mixed_moe_override_group_gets_routed_experts(self):+ """When MoE layers are in an *override* group (not the default ["Linear"]+ catch-all), the explicit targets list must include RoutedExperts."""+ # default=MXFP8, block_sparse_moe layers override to MXFP4+ # → they land in the override group with an explicit targets list.+ moe_layers = [+ "model.layers.0.block_sparse_moe.experts.0.w1",+ "model.layers.0.block_sparse_moe.experts.0.w3",+ ]+ dense_layers = [+ "model.layers.0.mlp.down_proj",+ "model.layers.0.mlp.up_proj",+ ]+ all_layers = moe_layers + dense_layers++ default_scheme = {"bits": 8, "data_type": "mx_fp", "group_size": 32}+ layer_cfg = {+ ".block_sparse_moe.": {"bits": 4, "data_type": "mx_fp", "group_size": 32},+ }+ qconfig = _build_mxfp_quantization_config(+ default_scheme=default_scheme,+ quantized_layers=all_layers,+ ignored_layers=[],+ layer_config=layer_cfg,+ )+ assert qconfig.get("format") == "mixed-precision"+ config_groups = qconfig.get("config_groups", {})+ # Find the override group (MXFP4, explicit layer list) containing MoE layers.+ moe_group_targets = None+ for grp in config_groups.values():+ t = grp.get("targets", [])+ if t != ["Linear"] and any("block_sparse_moe" in str(x) for x in t):+ moe_group_targets = t+ break+ assert moe_group_targets is not None, "Expected a non-default group for MoE layers"+ assert "RoutedExperts" in moe_group_targets, f"Missing RoutedExperts in: {moe_group_targets}"+++# ---------------------------------------------------------------------------+# _build_quantization_config: BF16-default + NVFP4 llm_compressor format+# TODO: Remove when the issue is fixed+# https://github.com/vllm-project/llm-compressor/issues/3069+# ---------------------------------------------------------------------------++# Only the w1/w3 layers that actually match the nv_fp layer_config are passed+# as quantized_layers (w2 stays at BF16 in this scenario).+_MOE_NV_QUANTIZED = [+ "model.layers.0.mlp.experts.0.w1",+ "model.layers.0.mlp.experts.0.w3",+ "model.layers.1.mlp.experts.0.w1",+ "model.layers.1.mlp.experts.0.w3",+]+++class TestBuildQuantizationConfigNvfp4MoeLlmCompressor:+ """RoutedExperts must be present when BF16-default with NVFP4 layer_config+ is exported in llm_compressor format and quantized layers are MoE.++ TODO: Remove when the issue is fixed+ https://github.com/vllm-project/llm-compressor/issues/3069+ """++ def test_nvfp4_moe_adds_routed_experts(self):+ default_scheme = {"bits": 16, "data_type": "float", "group_size": -1}+ layer_cfg = {+ ".w1.": {"bits": 4, "data_type": "nv_fp", "group_size": 16},+ ".w3.": {"bits": 4, "data_type": "nv_fp", "group_size": 16},+ }+ qconfig = _build_quantization_config(+ default_scheme=default_scheme,+ layer_config=layer_cfg,+ ignore_patterns=[],+ quantized_layers=_MOE_NV_QUANTIZED,+ ignored_layers=[],+ format="llm_compressor",+ )+ targets = _get_targets_from_qconfig(qconfig, "group_0")+ assert "RoutedExperts" in targets, f"Expected RoutedExperts in targets, got: {targets}"++ def test_nvfp4_dense_no_routed_experts(self):+ default_scheme = {"bits": 16, "data_type": "float", "group_size": -1}+ layer_cfg = {".down_proj.": {"bits": 4, "data_type": "nv_fp", "group_size": 16}}+ qconfig = _build_quantization_config(+ default_scheme=default_scheme,+ layer_config=layer_cfg,+ ignore_patterns=[],+ quantized_layers=["model.layers.0.mlp.down_proj"],+ ignored_layers=[],+ format="llm_compressor",+ )+ targets = _get_targets_from_qconfig(qconfig, "group_0")+ assert "RoutedExperts" not in targets, f"Unexpected RoutedExperts in targets: {targets}"+++# ===========================================================================+# _PatternMatcher+# ===========================================================================+++class TestPatternMatcher:+ def test_ignore_substring(self):+ m = _matcher(ignore=["mlp"])+ assert m.should_ignore("model.layers.0.mlp.fc1.weight") is True+ assert m.should_ignore("model.layers.0.self_attn.q_proj.weight") is False++ def test_ignore_trailing_dot(self):+ m = _matcher(ignore=["layers.4."])+ assert m.should_ignore("model.layers.4.mlp.fc1.weight") is True+ assert m.should_ignore("model.layers.45.mlp.fc1.weight") is False++ def test_skip_predefined(self):+ m = _matcher()+ assert m.should_skip("model.layers.0.shared_expert_gate.weight") is True+ assert m.should_skip("model.layers.0.mlp.gate.weight") is True+ assert m.should_skip("model.layers.0.mlp.gate_proj.weight") is False+ assert m.should_skip("model.embed_tokens.weight") is True+ assert m.should_skip("model.layers.0.mlp.fc1.weight") is False++ def test_resolve_scheme_exact_regex_and_default(self):+ default = {"bits": 4, "group_size": 128, "sym": True}+ lc = {+ "model.layers.0.mlp.fc1": {"bits": 8, "group_size": 32},+ r".*k_proj": {"bits": 8},+ }+ m = _matcher(layer_config=lc, default=default)+ assert m.resolve_scheme("model.layers.0.mlp.fc1.weight")["bits"] == 8+ assert m.resolve_scheme("model.layers.0.self_attn.k_proj.weight")["bits"] == 8+ assert m.resolve_scheme("model.layers.0.mlp.fc2.weight") == default++ def test_resolve_bits16_returns_none(self):+ m = _matcher(layer_config={"model.layers.0.fc1": {"bits": 16}}, default={"bits": 4, "group_size": 128})+ assert m.resolve_scheme("model.layers.0.fc1.weight") is None++ def test_resolve_substring_pattern(self):+ default = {"bits": 4, "group_size": 128, "sym": True}+ m = _matcher(layer_config={".ffn.experts.": {"bits": 2, "group_size": 64}}, default=default)+ r = m.resolve_scheme("model.layers.0.ffn.experts.3.gate_proj.weight")+ assert r["bits"] == 2 and r["group_size"] == 64+ assert m.resolve_scheme("model.layers.0.self_attn.q_proj.weight") == default+++# ===========================================================================+# _parse_layer_config — scheme key resolution+# ===========================================================================+++class TestParseLayerConfig:+ @staticmethod+ def _make_core(layer_config_input, scheme="W4A16"):+ core = _ModelFreeCompressorCore(+ model_name_or_path="dummy",+ output_dir="dummy_out",+ scheme=scheme,+ )+ core.layer_config_input = layer_config_input+ core._parse_scheme()+ core._parse_layer_config()+ return core++ def test_scheme_key_resolves(self):+ core = self._make_core({".ffn.experts.": {"scheme": "W2A16"}})+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)+ assert cfg["bits"] == 2 and "scheme" not in cfg++ m = _matcher(layer_config=core.layer_config, default=core.default_scheme)+ assert m.resolve_scheme("model.layers.0.ffn.experts.3.gate_proj.weight")["bits"] == 2++ def test_scheme_key_with_overrides(self):+ core = self._make_core({".ffn.experts.": {"scheme": "W2A16", "group_size": 32}})+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)+ assert cfg["bits"] == 2 and cfg["group_size"] == 32++ def test_string_value(self):+ core = self._make_core({".ffn.experts.": "W2A16"})+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)+ assert cfg["bits"] == 2++ def test_quantization_scheme_value(self):+ core = self._make_core({".ffn.experts.": QuantizationScheme(bits=2, group_size=64)})+ cfg = next(v for k, v in core.layer_config.items() if "ffn.experts" in k)+ assert cfg["bits"] == 2 and cfg["group_size"] == 64++ def test_w4a16_mixed_recipe_in_model_free(self):+ core = self._make_core({}, scheme="W4A16_MIXED")+ assert core.default_scheme["bits"] == 8+ assert core.layer_config[".experts."]["bits"] == 4+ assert core.layer_config[".moe."]["bits"] == 4+ assert core.layer_config[".shared_expert."]["bits"] == 8+++# ===========================================================================+# _build_quantization_config (generic INT path)+# ===========================================================================+++class TestBuildQuantizationConfig:+ def test_extra_config_filters_embed_conv_only(self):+ default = {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"}+ ignored = [+ "model.embed_tokens",+ "model.conv1",+ "model.layers.0.shared_expert_gate",+ "model.layers.0.mlp.gate",+ ]++ cfg = _build_quantization_config(+ default_scheme=default,+ layer_config={},+ ignore_patterns=[],+ quantized_layers=[],+ ignored_layers=ignored,+ )++ extra = cfg.get("extra_config", {})+ assert "model.embed_tokens" not in extra+ assert "model.conv1" not in extra+ assert extra["model.layers.0.shared_expert_gate"] == {"bits": 16, "data_type": "float"}+ assert extra["model.layers.0.mlp.gate"] == {"bits": 16, "data_type": "float"}+++# ===========================================================================+# _quantize_weight_mxfp+# ===========================================================================+++class TestQuantizeWeightMXFP:+ def test_disable_opt_rtn_skips_mxfp_scale_search(self, monkeypatch):+ from auto_round.data_type import mxfp++ search_mock = Mock(side_effect=AssertionError("optimized RTN must be disabled"))+ monkeypatch.setattr(mxfp, "quant_mx_opt_rtn", search_mock)++ weight = torch.randn(8, 32, dtype=torch.bfloat16)+ out = _quantize_weight_mxfp(+ weight,+ "layer",+ bits=4,+ group_size=32,+ data_type="mx_fp",+ disable_opt_rtn=True,+ )++ assert "layer.weight_packed" in out+ search_mock.assert_not_called()++ def test_quantize_weight_mxfp4_shapes(self):+ w = torch.randn(64, 128, dtype=torch.bfloat16)+ out = _quantize_weight_mxfp(w, "layer", bits=4, group_size=32, data_type="mx_fp")+ assert out["layer.weight_packed"].shape == (64, 64)+ assert out["layer.weight_packed"].dtype == torch.uint8+ assert out["layer.weight_scale"].shape == (64, 4)+ assert out["layer.weight_scale"].dtype == torch.uint8++ def test_quantize_weight_mxfp8_shapes(self):+ w = torch.randn(64, 128, dtype=torch.bfloat16)+ out = _quantize_weight_mxfp(w, "layer", bits=8, group_size=32, data_type="mx_fp")+ assert out["layer.weight"].shape == (64, 128)+ assert out["layer.weight"].dtype == torch.float8_e4m3fn+ assert out["layer.weight_scale"].shape == (64, 4)+ assert out["layer.weight_scale"].dtype == torch.uint8+++# ===========================================================================+# _build_mxfp_quantization_config (non-MoE, structural tests)+# ===========================================================================+++class TestBuildMxfpConfig:+ @require_compressed_tensors+ def test_build_mxfp_mixed_config_uniform(self):+ default = {"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"}+ quantized = ["model.layers.0.fc1", "model.layers.0.fc2"]+ ignored = ["lm_head"]+ cfg = _build_mxfp_quantization_config(default, quantized, ignored, layer_config={})+ assert cfg["format"] == "mxfp4-pack-quantized"+ assert "lm_head" in cfg["ignore"]+ assert len(cfg["config_groups"]) == 1++ @require_compressed_tensors+ def test_build_mxfp_config_bf16_default_targets_only_overrides(self):+ default = {"bits": 16, "group_size": -1, "sym": True, "data_type": "bf16"}+ layer_config = {+ "model.layers.0.self_attn.q_proj": {"bits": 4, "group_size": 32, "data_type": "mx_fp"},+ }+ quantized = ["model.layers.0.self_attn.q_proj"]+ cfg = _build_mxfp_quantization_config(default, quantized, [], layer_config=layer_config)+ assert cfg["format"] == "mxfp4-pack-quantized"+ assert cfg["config_groups"]["group_0"]["targets"] == quantized++ @require_compressed_tensors+ def test_build_mxfp_mixed_config_two_groups(self):+ default = {"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"}+ layer_config = {+ "model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 32, "data_type": "mx_fp"},+ "model.layers.0.self_attn.k_proj": {"bits": 8, "group_size": 32, "data_type": "mx_fp"},+ }+ quantized = [+ "model.layers.0.self_attn.q_proj",+ "model.layers.0.self_attn.k_proj",+ "model.layers.0.mlp.fc1",+ "model.layers.0.mlp.fc2",+ ]+ ignored = ["lm_head"]+ cfg = _build_mxfp_quantization_config(default, quantized, ignored, layer_config=layer_config)+ assert cfg["format"] == "mixed-precision"+ assert len(cfg["config_groups"]) == 2+ mxfp8_group = next(g for g in cfg["config_groups"].values() if g["format"] == "mxfp8-quantized")+ assert set(mxfp8_group["targets"]) == {+ "model.layers.0.self_attn.q_proj",+ "model.layers.0.self_attn.k_proj",+ }+ mxfp4_group = next(g for g in cfg["config_groups"].values() if g["format"] == "mxfp4-pack-quantized")+ assert mxfp4_group["targets"] == ["Linear"]+++# ===========================================================================+# _expand_e8m0_block_scale+# ===========================================================================+++class TestExpandE8M0BlockScale:+ def test_expand_repeat_interleave(self):+ scale = torch.tensor([[100, 101], [102, 103]], dtype=torch.uint8)+ out = _expand_e8m0_block_scale(scale, out_features=64, in_features=128, group_size=32)+ assert out.shape == (64, 4)+ assert out.dtype == torch.uint8+ assert out[0].tolist() == [100, 100, 101, 101]+ assert out[63].tolist() == [102, 102, 103, 103]++ def test_expand_noop_when_already_fine(self):+ scale = torch.full((64, 4), 127, dtype=torch.uint8)+ out = _expand_e8m0_block_scale(scale, out_features=64, in_features=128, group_size=32)+ assert out.shape == (64, 4) and torch.equal(out, scale)++ def test_expand_invalid_shape_raises(self):+ scale = torch.full((3, 4), 127, dtype=torch.uint8)+ with pytest.raises(ValueError):+ _expand_e8m0_block_scale(scale, out_features=64, in_features=128, group_size=32)+++# ===========================================================================+# _handle_mxfp_source_tensors / _dequant_mxfp_tensors+# ===========================================================================++_DEEPSEEK_V4_CFG = {"architectures": ["DeepseekV4ForCausalLM"], "model_type": "deepseek_v4"}+_LLAMA_CFG = {"architectures": ["LlamaForCausalLM"], "model_type": "llama"}+_DEFAULT_SCHEME = {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"}+_KIMI_K25_CFG = {+ "architectures": ["KimiK25ForConditionalGeneration"],+ "model_type": "kimi_k25",+ "quantization_config": {+ "quant_method": "compressed-tensors",+ "weights": {"num_bits": 4, "type": "int", "group_size": 8, "symmetric": True},+ },+}+_LLMCOMPRESSOR_MXFP_CFG_FP8 = {+ "architectures": ["Qwen3ForCausalLM"],+ "model_type": "qwen3",+ "quantization_config": {"quant_method": "compressed-tensors", "format": "mxfp8-quantized"},+}+_LLMCOMPRESSOR_MIXED_CFG = {+ "architectures": ["Qwen3ForCausalLM"],+ "model_type": "qwen3",+ "quantization_config": {"quant_method": "compressed-tensors", "format": "mixed-precision"},+}+++def _make_deepseek_v4_mxfp8(out_f, in_f, block_h, block_w):+ weight_fp8 = torch.randn(out_f, in_f, dtype=torch.bfloat16).to(torch.float8_e4m3fn)+ scale = torch.full((out_f // block_h, in_f // block_w), 127, dtype=torch.uint8)+ return weight_fp8, scale+++class TestHandleMXFPSourceTensors:+ def test_passthrough_mxfp8_same_target(self):+ weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)+ weight_scale = torch.full((64, 4), 127, dtype=torch.uint8)+ raw = {"layer.weight": weight_fp8.clone(), "layer.weight_scale": weight_scale.clone()}+ matcher = _matcher(default={"bits": 8, "group_size": 32, "sym": True, "data_type": "mx_fp"})+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)+ assert layers == ["layer"]+ assert passthrough["layer.weight"].dtype == torch.float8_e4m3fn+ assert torch.equal(passthrough["layer.weight"].view(torch.uint8), weight_fp8.view(torch.uint8))+ assert passthrough["layer.weight_scale"].dtype == torch.uint8+ assert "layer.weight" not in raw_out+ assert "layer.weight_scale" not in raw_out++ def test_passthrough_mxfp4_same_target(self):+ weight_packed = torch.randint(0, 255, (64, 64), dtype=torch.uint8)+ weight_scale = torch.full((64, 4), 127, dtype=torch.uint8)+ raw = {"layer.weight_packed": weight_packed.clone(), "layer.weight_scale": weight_scale.clone()}+ matcher = _matcher(default={"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"})+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)+ assert layers == ["layer"]+ assert torch.equal(passthrough["layer.weight_packed"], weight_packed)+ assert passthrough["layer.weight_scale"].dtype == torch.uint8+ assert "layer.weight_packed" not in raw_out++ def test_mxfp8_dequant_when_int_target(self):+ weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)+ weight_scale = torch.full((64, 4), 127, dtype=torch.uint8)+ raw = {"layer.weight": weight_fp8.clone(), "layer.weight_scale": weight_scale.clone()}+ matcher = _matcher(default={"bits": 4, "group_size": 128, "sym": True, "data_type": "int"})+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)+ assert layers == [] and passthrough == {}+ assert raw_out["layer.weight"].dtype == torch.bfloat16+ assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.bfloat16))+ assert "layer.weight_scale" not in raw_out++ def test_mixed_passthrough_and_dequant(self):+ weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)+ scale_fp8 = torch.full((64, 4), 127, dtype=torch.uint8)+ weight_packed = torch.randint(0, 255, (64, 64), dtype=torch.uint8)+ scale_packed = torch.full((64, 4), 127, dtype=torch.uint8)+ raw = {+ "attn.weight": weight_fp8.clone(),+ "attn.weight_scale": scale_fp8.clone(),+ "mlp.weight_packed": weight_packed.clone(),+ "mlp.weight_scale": scale_packed.clone(),+ }+ matcher = _matcher(default={"bits": 8, "group_size": 32, "sym": True, "data_type": "mx_fp"})+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)+ assert "attn" in layers+ assert passthrough["attn.weight"].dtype == torch.float8_e4m3fn+ assert "mlp" not in layers+ assert raw_out["mlp.weight"].dtype == torch.bfloat16+ assert raw_out["mlp.weight"].shape == (64, 128)++ def test_noop_without_mxfp_tensors(self):+ raw = {"layer.weight": torch.randn(64, 128, dtype=torch.bfloat16)}+ matcher = _matcher(default={"bits": 8, "group_size": 32, "data_type": "mx_fp"})+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)+ assert raw_out is raw and passthrough == {} and layers == []++ def test_reject_non_mxfp_packed_float_scale(self):+ raw = {+ "layer.weight_packed": torch.randint(0, 255, (64, 64), dtype=torch.uint8),+ "layer.weight_scale": torch.ones(64, 8, dtype=torch.float16),+ }+ matcher = _matcher(default={"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"})+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw, matcher)+ assert raw_out is raw and passthrough == {} and layers == []++ def test_convert_passthrough_when_target_mxfp8(self):+ weight_fp8, scale = _make_deepseek_v4_mxfp8(64, 128, block_h=32, block_w=64)+ raw = {"layer.weight": weight_fp8.clone(), "layer.scale": scale.clone()}+ matcher = _matcher(default={"bits": 8, "group_size": 32, "sym": True, "data_type": "mx_fp"})+ raw_out, state = _preprocess_model_type_source_tensors(raw, model_type="deepseek_v4")+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw_out, matcher, source_state=state)+ assert layers == ["layer"]+ assert passthrough["layer.weight"].dtype == torch.float8_e4m3fn+ assert torch.equal(passthrough["layer.weight"].view(torch.uint8), weight_fp8.view(torch.uint8))+ assert passthrough["layer.weight_scale"].dtype == torch.uint8+ assert passthrough["layer.weight_scale"].shape == (64, 4)+ assert "layer.weight" not in raw_out and "layer.scale" not in raw_out++ def test_convert_dequant_when_target_int(self):+ weight_fp8, scale = _make_deepseek_v4_mxfp8(64, 128, block_h=32, block_w=64)+ raw = {"layer.weight": weight_fp8.clone(), "layer.scale": scale.clone()}+ matcher = _matcher(default={"bits": 4, "group_size": 128, "sym": True, "data_type": "int"})+ raw_out, state = _preprocess_model_type_source_tensors(raw, model_type="deepseek_v4")+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw_out, matcher, source_state=state)+ assert layers == [] and passthrough == {}+ assert raw_out["layer.weight"].dtype == torch.bfloat16+ assert raw_out["layer.weight"].shape == (64, 128)+ assert torch.allclose(raw_out["layer.weight"], weight_fp8.to(torch.bfloat16))+ assert "layer.scale" not in raw_out and "layer.weight_scale" not in raw_out++ def test_convert_noop_without_quantized(self):+ raw = {"layer.weight": torch.randn(64, 128, dtype=torch.bfloat16)}+ matcher = _matcher(default={"bits": 8, "group_size": 32, "data_type": "mx_fp"})+ raw_out, state = _preprocess_model_type_source_tensors(raw, model_type="deepseek_v4")+ assert state == {}+ raw_out, passthrough, layers = _handle_mxfp_source_tensors(raw_out, matcher, source_state=state)+ assert raw_out is raw and passthrough == {} and layers == []++ def test_dequant_mxfp_tensors_mxfp8(self):+ weight_fp8 = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)+ weight_scale = torch.full((64, 4), 127, dtype=torch.uint8)+ raw = {"layer.weight": weight_fp8.clone(), "layer.weight_scale": weight_scale.clone()}+ out = _dequant_mxfp_tensors(raw)+ assert out["layer.weight"].dtype == torch.bfloat16+ assert torch.allclose(out["layer.weight"], weight_fp8.to(torch.bfloat16))+ assert "layer.weight_scale" not in out+++# ===========================================================================+# AutoScheme helpers+# ===========================================================================+++class TestAutoSchemeHelpers:+ def test_looks_like_auto_scheme(self):+ from auto_round import AutoScheme++ assert _looks_like_auto_scheme(AutoScheme(avg_bits=3, options=("W2A16", "W4A16")))+ assert not _looks_like_auto_scheme("W4A16")+ assert not _looks_like_auto_scheme(QuantizationScheme(bits=4))++ def test_validate_options_int_family(self):+ from auto_round import AutoScheme++ assert _validate_auto_scheme_options(AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "W8A16"))) == "int"++ def test_validate_options_int_family_with_bf16(self):+ from auto_round import AutoScheme++ assert _validate_auto_scheme_options(AutoScheme(avg_bits=3, options=("W2A16", "W4A16", "BF16"))) == "int"++ def test_validate_options_mxfp_family(self):+ from auto_round import AutoScheme++ assert _validate_auto_scheme_options(AutoScheme(avg_bits=6, options=("MXFP4", "MXFP8"))) == "mx_fp"++ def test_validate_options_mixed_family_raises(self):+ from auto_round import AutoScheme++ with pytest.raises(ValueError, match="mix INT and MXFP"):+ _validate_auto_scheme_options(AutoScheme(avg_bits=4, options=("W4A16", "MXFP4")))++ @pytest.mark.parametrize(+ "options",+ [+ ("W3A16", "W4A16"),+ ("GGUF:Q4_K_M", "W8A16"),+ ("NVFP4", "W4A16"),+ ("MXFP4_RCEIL", "MXFP4"),+ ],+ )+ def test_validate_options_unsupported_raises(self, options):+ from auto_round import AutoScheme++ with pytest.raises(ValueError, match="unsupported option"):+ _validate_auto_scheme_options(AutoScheme(avg_bits=4, options=options))++ def test_convert_layer_config(self):+ generated = {+ "model.layers.0.q_proj": {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"},+ "model.layers.0.k_proj": {"bits": 2, "group_size": 128, "sym": True, "data_type": "int"},+ "model.layers.0.v_proj": {"bits": 4, "group_size": 128, "sym": True, "data_type": "int"},+ "model.embed_tokens": {"bits": 16, "group_size": 128, "sym": True, "data_type": "int"},+ }+ base_scheme, per_layer, fp16_layers = _convert_auto_scheme_layer_config(generated)+ assert base_scheme.bits == 4 and base_scheme.group_size == 128+ assert per_layer["model.layers.0.k_proj"]["bits"] == 2+ assert "model.embed_tokens" not in per_layer+ assert fp16_layers == ["model.embed_tokens"]++ def test_convert_layer_config_infers_mxfp_bits_from_dtype_alias(self):+ generated = {+ "model.layers.0.q_proj": {"group_size": 32, "sym": True, "data_type": "mxfp8"},+ "model.layers.0.k_proj": {"group_size": 32, "sym": True, "data_type": "MXFP4"},+ }+ base_scheme, per_layer, fp16_layers = _convert_auto_scheme_layer_config(generated)+ assert fp16_layers == []+ assert per_layer["model.layers.0.q_proj"]["bits"] == 8+ assert per_layer["model.layers.0.q_proj"]["data_type"] == "mx_fp"+ assert per_layer["model.layers.0.k_proj"]["bits"] == 4+ assert per_layer["model.layers.0.k_proj"]["data_type"] == "mx_fp"+ assert base_scheme.data_type == "mx_fp"+ assert base_scheme.bits in (4, 8)+++# ===========================================================================+# _build_mxfp_autoround_quantization_config+# ===========================================================================+++class TestBuildMxfpAutoRoundConfig:+ @require_compressed_tensors+ def test_mxfp4_top_level_fields(self):+ from dataclasses import asdict++ from auto_round.schemes import PRESET_SCHEMES++ default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP4"]).items() if v is not None}+ cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])+ assert cfg["quant_method"] == "auto-round"+ assert cfg["packing_format"] == "auto_round:llm_compressor"+ assert cfg["bits"] == 4+ assert cfg["group_size"] == 32+ assert cfg["data_type"] == "mx_fp"+ assert cfg["sym"] is True+ assert cfg["enable_quanted_input"] is False+ assert cfg["model_free"] is True+ assert "autoround_version" in cfg++ @require_compressed_tensors+ def test_mxfp4_activation_fields(self):+ from dataclasses import asdict++ from auto_round.schemes import PRESET_SCHEMES++ default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP4"]).items() if v is not None}+ cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])+ assert cfg["act_bits"] == 4+ assert cfg["act_data_type"] == "mx_fp"+ assert cfg["act_dynamic"] is True+ assert cfg["act_group_size"] == 32+ assert cfg["act_sym"] is True++ @require_compressed_tensors+ def test_mxfp8_bits(self):+ from dataclasses import asdict++ from auto_round.schemes import PRESET_SCHEMES++ default = {k: v for k, v in asdict(PRESET_SCHEMES["MXFP8"]).items() if v is not None}+ cfg = _build_mxfp_autoround_quantization_config(default, quantized_layers=["layer.fc1"], ignored_layers=[])+ assert cfg["bits"] == 8+ assert cfg["act_bits"] == 8+ assert cfg["packing_format"] == "auto_round:llm_compressor"++ @require_compressed_tensors+ def test_ignored_layers_in_extra_config(self):+ default = {+ "bits": 4,+ "group_size": 32,+ "sym": True,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_data_type": "mx_fp",+ "act_dynamic": True,+ "act_group_size": 32,+ "act_sym": True,+ }+ ignored = ["lm_head", "model.embed_tokens", "model.conv1", "model.layers.0.mlp.gate"]+ cfg = _build_mxfp_autoround_quantization_config(+ default,+ quantized_layers=["model.layers.0.fc1"],+ ignored_layers=ignored,+ )+ extra = cfg.get("extra_config", {})+ full_precision = {"bits": 16, "data_type": "float", "act_bits": 16, "act_data_type": "float"}+ assert extra.get("lm_head") == full_precision+ assert extra.get("model.layers.0.mlp.gate") == full_precision+ assert "model.embed_tokens" not in extra+ assert "model.conv1" not in extra++ @require_compressed_tensors+ def test_quantized_lm_head_in_extra_config(self):+ default = {+ "bits": 4,+ "group_size": 32,+ "sym": True,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_data_type": "mx_fp",+ "act_dynamic": True,+ "act_group_size": 32,+ "act_sym": True,+ }+ cfg = _build_mxfp_autoround_quantization_config(+ default,+ quantized_layers=["model.layers.0.fc1", "lm_head"],+ ignored_layers=[],+ )+ extra = cfg.get("extra_config", {})+ assert "lm_head" in extra+ assert extra["lm_head"]["bits"] == 4+ assert extra["lm_head"]["data_type"] == "mx_fp"++ @require_compressed_tensors+ def test_build_quantization_config_routes_mxfp_to_autoround(self):+ default = {+ "bits": 4,+ "group_size": 32,+ "sym": True,+ "data_type": "mx_fp",+ "act_bits": 4,+ "act_data_type": "mx_fp",+ "act_dynamic": True,+ "act_group_size": 32,+ "act_sym": True,+ }+ cfg = _build_quantization_config(+ default_scheme=default,+ layer_config={},+ ignore_patterns=[],+ quantized_layers=["layer.fc1"],+ ignored_layers=[],+ format="auto_round",+ )+ assert cfg["quant_method"] == "auto-round"+ assert cfg["packing_format"] == "auto_round:llm_compressor"++ @require_compressed_tensors+ def test_build_quantization_config_mxfp_llmcompressor_path_unchanged(self):+ default = {"bits": 4, "group_size": 32, "sym": True, "data_type": "mx_fp"}+ cfg = _build_quantization_config(+ default_scheme=default,+ layer_config={},+ ignore_patterns=[],+ quantized_layers=["layer.fc1"],+ ignored_layers=["lm_head"],+ format="llm_compressor",+ )+ assert cfg["quant_method"] == "compressed-tensors"+ assert cfg["format"] == "mxfp4-pack-quantized"+++# ===========================================================================+# is_model_free_route (basic function tests)+# ===========================================================================++[email protected]("scheme", ["MXFP8", "mxfp8"])+def test_model_free_route_accepts_mxfp_scheme_case_insensitively(scheme):+ assert is_model_free_route("model", scheme, 0, True, {})++[email protected]("dtype_key", ["static_kv_dtype", "static_attention_dtype"])[email protected]("explicit", [False, True])+def test_model_free_route_rejects_static_attention_quantization(dtype_key, explicit):+ kwargs = {dtype_key: "fp8", "model_free": explicit}+ assert not is_model_free_route("model", "W4A16", 0, True, kwargs)+++# ===========================================================================+# _resolve_model_type (model-type unit tests)+# ===========================================================================+++class TestResolveModelTypeDeepseekV4:+ """_resolve_model_type unit tests for deepseek_v4."""++ def test_resolve_model_type(self):+ core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")+ core.config = _DEEPSEEK_V4_CFG+ core._resolve_model_type()+ assert core.model_type == "deepseek_v4"++ def test_resolve_model_type_negative(self):+ core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")+ core.config = _LLAMA_CFG+ core._resolve_model_type()+ assert core.model_type == "llama"+++class TestResolveModelTypeLLMCompressor:+ """_resolve_model_type unit tests for llm-compressor MXFP models."""++ def test_resolve_model_type_qwen3(self):+ core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")+ core.config = _LLMCOMPRESSOR_MXFP_CFG_FP8+ core._resolve_model_type()+ assert core.model_type == "qwen3"++ def test_resolve_model_type_mixed(self):+ core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")+ core.config = _LLMCOMPRESSOR_MIXED_CFG+ core._resolve_model_type()+ assert core.model_type == "qwen3"++ def test_resolve_model_type_negative_not_compressed_tensors(self):+ core = _ModelFreeCompressorCore(model_name_or_path="x", output_dir="o", scheme="MXFP8")+ core.config = _LLAMA_CFG+ core._resolve_model_type()+ assert core.model_type == "llama"+++# ===========================================================================+# Predefined ignore layers+# ===========================================================================+++class TestPredefinedIgnoreLayers:+ def test_normal_model_empty(self):+ assert get_predefined_ignore_layers_from_config({"architectures": ["LlamaForCausalLM"]}) == []++ def test_step3p5_ignore_layers(self):+ cfg = {"model_type": "step3p5"}+ assert get_predefined_ignore_layers_from_config(cfg) == [+ "g_proj",+ "moe.gate",+ "eh_proj",+ "shared_head",+ "layers.45",+ ]+++# ===========================================================================+# Cross-shard FP8 scale handling+# ===========================================================================+++def _write_fake_fp8_shard(path: str, tensors: dict) -> None:+ """Save a dict of tensors as a safetensors file."""+ save_file(tensors, path)+++def _write_index_json(directory: str, weight_map: dict) -> str:+ index_path = os.path.join(directory, "model.safetensors.index.json")+ with open(index_path, "w") as f:+ json.dump({"weight_map": weight_map}, f)+ return index_path+++class TestBuildCrossShardPairs:+ """Tests for _build_cross_shard_pairs_from_weight_map."""++ def test_no_fp8_entries_returns_empty(self):+ weight_map = {+ "model.layer.weight": "shard-00001.safetensors",+ "model.layer.bias": "shard-00001.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)+ assert recipient_to_donors == {}+ assert donor_shard_tensors == {}++ def test_same_shard_scale_not_cross(self):+ """weight and weight_scale_inv in the same shard → not a cross-shard pair."""+ weight_map = {+ "model.layer.weight": "shard-00001.safetensors",+ "model.layer.weight_scale_inv": "shard-00001.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)+ assert recipient_to_donors == {}+ assert donor_shard_tensors == {}++ def test_cross_shard_single_pair(self):+ """weight in shard-1, weight_scale_inv in shard-2."""+ weight_map = {+ "model.layer.weight": "shard-00001.safetensors",+ "model.layer.weight_scale_inv": "shard-00002.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)++ assert "shard-00001.safetensors" in recipient_to_donors+ donor_map = recipient_to_donors["shard-00001.safetensors"]+ assert "shard-00002.safetensors" in donor_map+ assert "model.layer.weight_scale_inv" in donor_map["shard-00002.safetensors"]++ assert "shard-00002.safetensors" in donor_shard_tensors+ assert "model.layer.weight_scale_inv" in donor_shard_tensors["shard-00002.safetensors"]++ def test_cross_shard_multiple_layers_one_donor(self):+ """Multiple layers whose scale_inv all live in the same donor shard."""+ weight_map = {+ "model.layer0.weight": "shard-00001.safetensors",+ "model.layer0.weight_scale_inv": "shard-00002.safetensors",+ "model.layer1.weight": "shard-00001.safetensors",+ "model.layer1.weight_scale_inv": "shard-00002.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)++ donor_map = recipient_to_donors["shard-00001.safetensors"]+ scales = donor_map["shard-00002.safetensors"]+ assert "model.layer0.weight_scale_inv" in scales+ assert "model.layer1.weight_scale_inv" in scales+ assert len(donor_shard_tensors["shard-00002.safetensors"]) == 2++ def test_cross_shard_multiple_donors(self):+ """Recipient shard needs scales from two different donor shards."""+ weight_map = {+ "model.layerA.weight": "shard-00001.safetensors",+ "model.layerA.weight_scale_inv": "shard-00002.safetensors",+ "model.layerB.weight": "shard-00001.safetensors",+ "model.layerB.weight_scale_inv": "shard-00003.safetensors",+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)++ donor_map = recipient_to_donors["shard-00001.safetensors"]+ assert "shard-00002.safetensors" in donor_map+ assert "shard-00003.safetensors" in donor_map+ assert len(donor_shard_tensors) == 2++ def test_scale_inv_without_matching_weight_ignored(self):+ """weight_scale_inv present in weight_map but no corresponding .weight → ignored."""+ weight_map = {+ "model.layer.weight_scale_inv": "shard-00002.safetensors",+ # no "model.layer.weight" key at all+ }+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map(weight_map)+ assert recipient_to_donors == {}+ assert donor_shard_tensors == {}++ def test_empty_weight_map(self):+ recipient_to_donors, donor_shard_tensors = _build_cross_shard_pairs_from_weight_map({})+ assert recipient_to_donors == {}+ assert donor_shard_tensors == {}+++class TestHydrateMissingFp8Scales:+ """Tests for _hydrate_missing_fp8_scales_from_index."""++ def test_non_safetensors_shard_returns_unchanged(self, tmp_path):+ raw = {"w": torch.zeros(4)}+ result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / "model.bin"))+ assert result is raw++ def test_no_fp8_weights_returns_unchanged(self, tmp_path):+ shard_path = str(tmp_path / "shard-00001.safetensors")+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.bfloat16)}+ result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)+ assert result is raw++ def test_all_scales_present_returns_unchanged(self, tmp_path):+ shard_path = str(tmp_path / "shard-00001.safetensors")+ raw = {+ "model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),+ "model.layer.weight_scale_inv": torch.ones(1),+ }+ result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)+ assert "model.layer.weight_scale_inv" in result++ def test_cross_shard_hydration_local_mode(self, tmp_path):+ """Recipient shard gets scale_inv from donor shard in local (non-streaming) mode."""+ donor_name = "shard-00002.safetensors"+ recipient_name = "shard-00001.safetensors"+ scale_name = "model.layer.weight_scale_inv"++ donor_path = tmp_path / donor_name+ _write_fake_fp8_shard(str(donor_path), {scale_name: torch.ones(1)})++ weight_map = {+ "model.layer.weight": recipient_name,+ scale_name: donor_name,+ }+ _write_index_json(str(tmp_path), weight_map)++ recipient_path = tmp_path / recipient_name+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}++ result = _hydrate_missing_fp8_scales_from_index(raw, str(recipient_path))+ assert scale_name in result, "scale_inv should be hydrated from donor shard"+ assert result[scale_name].dtype == torch.float32 or result[scale_name].numel() == 1++ def test_cross_shard_hydration_streaming_mode(self, tmp_path):+ """In streaming mode, index.json lives in work_dir, shards in cache subdir."""+ work_dir = tmp_path / "work_dir"+ cache_dir = work_dir / ".cache" / "model_free_source_shards"+ work_dir.mkdir()+ cache_dir.mkdir(parents=True)++ donor_name = "shard-00002.safetensors"+ recipient_name = "shard-00001.safetensors"+ scale_name = "model.layer.weight_scale_inv"++ _write_fake_fp8_shard(str(cache_dir / donor_name), {scale_name: torch.ones(1)})++ weight_map = {+ "model.layer.weight": recipient_name,+ scale_name: donor_name,+ }+ _write_index_json(str(work_dir), weight_map)++ recipient_path = cache_dir / recipient_name+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}++ result = _hydrate_missing_fp8_scales_from_index(+ raw,+ str(recipient_path),+ index_dir=str(work_dir),+ donor_shard_dir=str(cache_dir),+ )+ assert scale_name in result, "streaming mode: scale_inv should be hydrated via index_dir/donor_shard_dir params"++ def test_missing_index_json_returns_unchanged(self, tmp_path):+ """If no index.json exists, raw_tensors is returned as-is (no crash)."""+ shard_path = str(tmp_path / "shard-00001.safetensors")+ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}+ result = _hydrate_missing_fp8_scales_from_index(raw, shard_path)+ assert result is raw++ def test_donor_shard_missing_on_disk_returns_unchanged(self, tmp_path):+ """Index references a donor shard that doesn't exist on disk → graceful skip."""+ scale_name = "model.layer.weight_scale_inv"+ recipient_name = "shard-00001.safetensors"+ donor_name = "shard-00002.safetensors" # not written++ weight_map = {+ "model.layer.weight": recipient_name,+ scale_name: donor_name,+ }+ _write_index_json(str(tmp_path), weight_map)++ raw = {"model.layer.weight": torch.zeros(4, dtype=torch.float8_e4m3fn)}+ result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / recipient_name))+ assert scale_name not in result # hydration skipped silently++ def test_multiple_layers_hydrated_from_single_donor(self, tmp_path):+ """Multiple missing scale_inv tensors are hydrated in a single donor open."""+ donor_name = "shard-00002.safetensors"+ recipient_name = "shard-00001.safetensors"+ scales = {+ "model.layerA.weight_scale_inv": torch.ones(1),+ "model.layerB.weight_scale_inv": torch.ones(1),+ }+ _write_fake_fp8_shard(str(tmp_path / donor_name), scales)++ weight_map = {+ "model.layerA.weight": recipient_name,+ "model.layerA.weight_scale_inv": donor_name,+ "model.layerB.weight": recipient_name,+ "model.layerB.weight_scale_inv": donor_name,+ }+ _write_index_json(str(tmp_path), weight_map)++ raw = {+ "model.layerA.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),+ "model.layerB.weight": torch.zeros(4, dtype=torch.float8_e4m3fn),+ }+ result = _hydrate_missing_fp8_scales_from_index(raw, str(tmp_path / recipient_name))+ assert "model.layerA.weight_scale_inv" in result+ assert "model.layerB.weight_scale_inv" in result+++# ===========================================================================+# FP8 source model+# ===========================================================================+++class TestFP8Source:+ def test_dequant_fp8(self):+ w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)+ raw = {"layer.weight": w, "layer.weight_scale_inv": torch.tensor(0.5), "layer.bias": torch.randn(64)}+ result = _dequant_fp8_tensors(raw, block_size=None)+ assert result["layer.weight"].dtype == torch.bfloat16 and "layer.weight_scale_inv" not in result++ def test_no_fp8_noop(self):+ raw = {"layer.weight": torch.randn(64, 128)}+ assert _dequant_fp8_tensors(raw, block_size=None) is raw++ def test_process_shard_fp8(self, tmp_path):+ shard_path = str(tmp_path / "shard.safetensors")+ w = torch.randn(64, 128, dtype=torch.bfloat16).to(torch.float8_e4m3fn)+ save_file({"layer.weight": w, "layer.weight_scale_inv": torch.tensor(1.0)}, shard_path)+ output, quantized, _ = _process_shard(shard_path, _DEFAULT_SCHEME, {}, [], device="cpu", fp8_block_size=None)+ assert "layer" in quantized and "layer.qweight" in output++ def test_dequant_fp8_hydrates_scale_from_sibling_shard(self, tmp_path):+ """When scale_inv is sharded separately, dequant should hydrate it via index."""+ shard_dir = tmp_path / "source"+ shard_dir.mkdir(parents=True, exist_ok=True)++ weight_name = "model.layers.0.mlp.experts.1.gate_proj.weight"+ scale_name = "model.layers.0.mlp.experts.1.gate_proj.weight_scale_inv"++ shard_a = shard_dir / "model-00001-of-00002.safetensors"+ shard_b = shard_dir / "model-00002-of-00002.safetensors"+ save_file({weight_name: torch.randn(2048, 7168, dtype=torch.bfloat16).to(torch.float8_e4m3fn)}, str(shard_a))+ save_file({scale_name: torch.ones(16, 56, dtype=torch.float32)}, str(shard_b))++ with open(shard_dir / "model.safetensors.index.json", "w") as f:+ json.dump(+ {+ "metadata": {"total_size": 0},+ "weight_map": {+ weight_name: shard_a.name,+ scale_name: shard_b.name,+ },+ },+ f,+ )++ output, quantized, _ = _process_shard(+ str(shard_a),+ _DEFAULT_SCHEME,+ {},+ [],+ fp8_block_size=[128, 128],+ )+ assert "model.layers.0.mlp.experts.1.gate_proj" in quantized+ assert "model.layers.0.mlp.experts.1.gate_proj.qweight" in output+++# ===========================================================================+# kimi_k25 INT4 packed source models+# ===========================================================================+++class TestKimiK25Int4Source:+ def test_kimi_k25_int4_dequant_helper(self):+ raw = {+ "layer.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),+ "layer.weight_scale": torch.ones(128, 16, dtype=torch.float16),+ }+ out = _handle_model_type_low_precision_source_tensors(+ raw,+ model_type="kimi_k25",+ source_quant_config=_KIMI_K25_CFG["quantization_config"],+ device="cpu",+ )+ assert "layer.weight" in out+ assert out["layer.weight"].dtype == torch.bfloat16+ assert out["layer.weight"].shape == (128, 128)+ assert "layer.weight_packed" not in out+ assert "layer.weight_scale" not in out++ @require_compressed_tensors+ def test_kimi_k25_int4_to_mxfp4_via_model_free(self, tmp_path):+ tensors = {+ "model.layers.0.mlp.fc1.weight_packed": torch.randint(0, 255, (128, 64), dtype=torch.uint8),+ "model.layers.0.mlp.fc1.weight_scale": torch.ones(128, 16, dtype=torch.float16),+ "lm_head.weight": torch.randn(1000, 128),+ }+ model_dir = _make_model_dir(tmp_path, _KIMI_K25_CFG, tensors)+ output_dir = str(tmp_path / "output")++ _ModelFreeCompressorCore(+ model_name_or_path=model_dir,+ output_dir=output_dir,+ scheme="MXFP4",+ format="llm_compressor",+ ).run()++ qc = _read_qconfig(output_dir)+ assert qc["format"] == "mxfp4-pack-quantized"+ assert qc["quant_method"] == "compressed-tensors"++ found_scale_dtype = None+ found_packed_shape = None+ for fname in os.listdir(output_dir):+ if not fname.endswith(".safetensors"):+ continue+ with safe_open(os.path.join(output_dir, fname), framework="pt") as sf:+ if "model.layers.0.mlp.fc1.weight_scale" in sf.keys():+ found_scale_dtype = sf.get_tensor("model.layers.0.mlp.fc1.weight_scale").dtype+ found_packed_shape = sf.get_tensor("model.layers.0.mlp.fc1.weight_packed").shape++ # Re-quantized MXFP4 scales are uint8 E8M0 (source INT4 scales were fp16).+ assert found_scale_dtype == torch.uint8+ assert found_packed_shape == (128, 64)