Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

7.4 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""End-to-end MoE (Mixture of Experts) quantization tests.

These tests target the architectures that are most active in 2025-2026:
Qwen3-MoE, DeepSeek-V2-Lite, Mixtral, Llama-4, and OpenAI's
gpt-oss-20b (which uses native MXFP4).

The test pipeline is the canonical one:

    1. ``AutoRound`` quantizes the model.
    2. The saved checkpoint is reloaded.
    3. A single short generation is run.
    4. We assert the model produces non-garbage output and that
       ``model.config.num_local_experts`` is preserved through the
       save/load round-trip.

MoE model downloads are large; the cases are gated on free RAM.
"""

from __future__ import annotations

import os
import time
from test.e2e.test_cpu.conftest import (  # noqa: E402
    EvalResult,
    assert_non_garbage_output,
    quantize_and_save,
    record,
)

import pytest
import torch

# ---------------------------------------------------------------------------
# Matrix
# ---------------------------------------------------------------------------

# (hf_id, scheme, min_ram_gib, ignore)
# ``ignore`` matches the standard auto-round ignore list: router, lm_head,
# and the gating networks are left in fp16.
MOE_CASES = [
    ("Qwen/Qwen1.5-MoE-A2.7B", "W4A16", 16, "self_attn,router,lm_head,mlp.gate"),
    ("deepseek-ai/DeepSeek-V2-Lite-Chat", "W4A16", 16, "self_attn,router,lm_head,mlp.gate"),
    ("openai/gpt-oss-20b", "MXFP4", 24, "self_attn,lm_head"),
    # Mixtral and the bigger Qwen3-MoE are gated - the test will skip if
    # the host can't authenticate.  Listed for documentation, may be
    # enabled in CI when the org has a HF_TOKEN with the right scope.
    # ("mistralai/Mixtral-8x7B-Instruct-v0.1", "W4A16", 80, "self_attn,router,lm_head,mlp.gate"),
]


def _case_id(model_id: str, scheme: str) -> str:
    return f"{model_id.split('/')[-1].lower()}-{scheme.lower()}"


# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------


class TestMoeE2E:
    """Quantize + reload + generate for MoE architectures."""

    @pytest.mark.parametrize(
        "model_id,scheme,min_ram,ignore",
        MOE_CASES,
        ids=[_case_id(m, s) for m, s, *_ in MOE_CASES],
    )
    def test_quantize_and_generate(self, model_id, scheme, min_ram, ignore, tmp_path):
        import psutil  # type: ignore

        avail = psutil.virtual_memory().available / 1024**3
        if avail < min_ram:
            pytest.skip(f"only {avail:.1f} GiB free RAM, need {min_ram} GiB for {model_id}")

        from test.helpers import get_model_path

        model_id = get_model_path(model_id)
        save_dir = str(tmp_path / "moe_out")

        # ---- 1. quantize + save ----
        t0 = time.perf_counter()
        saved = quantize_and_save(
            model_id=model_id,
            bits=4 if "W4" in scheme else 0,
            group_size=128,
            sym=True,
            fmt="auto_round",
            output_dir=save_dir,
            iters=2,  # MoE models are slow; we keep the iter count small
            nsamples=4,
            seqlen=512,
            extra_kwargs={"ignore_layers": ignore, "disable_opt_rtn": False},
        )
        quant_time = time.perf_counter() - t0

        # ---- 2. reload + assert expert count is preserved ----
        from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer

        cfg = AutoConfig.from_pretrained(saved, trust_remote_code=True)
        orig_cfg = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
        # The number of experts must be preserved (the routers are not
        # quantized, but the per-expert weights are - and there must be
        # the same number of them after load).
        for attr in ("num_local_experts", "num_experts"):
            if hasattr(orig_cfg, attr):
                assert getattr(cfg, attr) == getattr(orig_cfg, attr), (
                    f"{model_id}: {attr} changed from "
                    f"{getattr(orig_cfg, attr)} to {getattr(cfg, attr)} after quantize+save"
                )

        # ---- 3. generate a few tokens ----
        tokenizer = AutoTokenizer.from_pretrained(saved, trust_remote_code=True)
        try:
            model = AutoModelForCausalLM.from_pretrained(saved, torch_dtype=torch.bfloat16)
        except Exception:
            # Some MoE exports may require specific dtypes.
            model = AutoModelForCausalLM.from_pretrained(saved)

        try:
            inputs = tokenizer("The capital of France is", return_tensors="pt")
            t0 = time.perf_counter()
            with torch.no_grad():
                ids = model.generate(**inputs, max_new_tokens=8, do_sample=False)
            gen_time = time.perf_counter() - t0
            text = tokenizer.decode(ids[0], skip_special_tokens=True)
            assert_non_garbage_output(text)
        finally:
            del model
            import gc

            gc.collect()

        record(
            EvalResult(
                test=self.__class__.__name__,
                model=model_id,
                fmt="auto_round",
                bits=4 if "W4" in scheme else 0,
                group_size=128,
                sym=True,
                task="generate",
                metric="gen_len",
                value=float(len(text.split())),
                wall_time_s=quant_time + gen_time,
                extra={"quant_time_s": quant_time, "gen_time_s": gen_time, "ignore_layers": ignore},
            )
        )
        print(f"\n[MoE-e2e] {model_id} {scheme} -> text={text!r}")


class TestMoeExpertUnfuse:
    """Verify the saved MoE checkpoint exposes per-expert modules.

    Several backends (vLLM, Marlin, ...) want the experts in their
    *fused* or *unfused* form depending on the model.  This test
    sanity-checks that the saved checkpoint has the right structure.
    """

    @pytest.mark.parametrize(
        "model_id,expected_expert_attr",
        [
            ("Qwen/Qwen1.5-MoE-A2.7B", "num_local_experts"),
        ],
        ids=["qwen-moe"],
    )
    def test_save_preserves_expert_count(self, model_id, expected_expert_attr, tmp_path):
        from test.helpers import get_model_path

        from transformers import AutoConfig

        model_id = get_model_path(model_id)

        save_dir = str(tmp_path / "moe_struct_out")
        quantize_and_save(
            model_id=model_id,
            bits=4,
            group_size=128,
            sym=True,
            fmt="auto_round",
            output_dir=save_dir,
            iters=1,
            nsamples=2,
            seqlen=128,
            extra_kwargs={"ignore_layers": "self_attn,router,lm_head,mlp.gate"},
        )

        cfg = AutoConfig.from_pretrained(save_dir, trust_remote_code=True)
        n_experts = getattr(cfg, expected_expert_attr, None)
        assert (
            n_experts is not None and n_experts > 0
        ), f"saved MoE checkpoint lost its expert count (cfg.{expected_expert_attr}={n_experts})"