Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

8.4 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytest
import torch
from torch import nn

from auto_round.modeling.fused_moe.fusion_spec import (
    MoEFusionSpec,
    MoETensorSource,
    ProjectionFusionSpec,
    build_standard_moe_fusion_spec,
    get_moe_fusion_spec,
    iter_moe_fusion_views,
    register_moe_fusion_spec,
)


class _Expert(nn.Module):
    def __init__(self, expert_index: int, bias: bool = False):
        super().__init__()
        self.gate_proj = nn.Linear(3, 2, bias=bias)
        self.up_proj = nn.Linear(3, 2, bias=bias)
        self.down_proj = nn.Linear(2, 3, bias=bias)

        offset = expert_index * 100
        with torch.no_grad():
            self.gate_proj.weight.copy_(torch.arange(6).reshape(2, 3) + offset)
            self.up_proj.weight.copy_(torch.arange(6).reshape(2, 3) + offset + 10)
            self.down_proj.weight.copy_(torch.arange(6).reshape(3, 2) + offset + 20)
            if bias:
                self.gate_proj.bias.copy_(torch.arange(2) + offset + 30)
                self.up_proj.bias.copy_(torch.arange(2) + offset + 40)
                self.down_proj.bias.copy_(torch.arange(3) + offset + 50)


class _Experts(nn.ModuleList):
    pass


class _Layer(nn.Module):
    def __init__(self, bias: bool = False):
        super().__init__()
        self.experts = _Experts([_Expert(0, bias=bias), _Expert(1, bias=bias)])


class _Model(nn.Module):
    def __init__(self, bias: bool = False):
        super().__init__()
        self.layers = nn.ModuleList([_Layer(bias=bias)])


def _standard_spec(model: _Model, checkpoint_transposed: bool = False):
    detected_projections = {
        "gate_up_proj": {"split_into": ["gate_proj", "up_proj"]},
        "down_proj": {},
    }
    return build_standard_moe_fusion_spec(
        detected_projections=detected_projections,
        num_experts=2,
        checkpoint_transposed=checkpoint_transposed,
        module=model.layers[0].experts,
    )


def test_non_transposed_views_reconstruct_checkpoint_without_mutating_model():
    model = _Model()
    experts = model.layers[0].experts
    spec = _standard_spec(model)
    register_moe_fusion_spec(experts, spec)

    gate_up = torch.stack(
        [torch.cat([experts[i].gate_proj.weight, experts[i].up_proj.weight], dim=0) for i in range(2)], dim=0
    )
    down = torch.stack([experts[i].down_proj.weight for i in range(2)], dim=0)

    views = {view.checkpoint_name: view for view in iter_moe_fusion_views(model)}

    assert torch.equal(views["layers.0.experts.gate_up_proj"].tensor_fn(), gate_up)
    assert torch.equal(views["layers.0.experts.down_proj"].tensor_fn(), down)
    assert views["layers.0.experts.gate_up_proj"].sources == (
        MoETensorSource(
            projection="gate_proj",
            hf_names=(
                "layers.0.experts.0.gate_proj.weight",
                "layers.0.experts.1.gate_proj.weight",
            ),
        ),
        MoETensorSource(
            projection="up_proj",
            hf_names=(
                "layers.0.experts.0.up_proj.weight",
                "layers.0.experts.1.up_proj.weight",
            ),
        ),
    )
    assert tuple(experts._modules) == ("0", "1")
    assert "gate_up_proj" not in experts._parameters
    assert "_auto_round_moe_fusion_spec" not in experts._modules
    assert "_auto_round_moe_fusion_spec" not in experts._parameters
    assert "_auto_round_moe_fusion_spec" not in experts._buffers
    assert get_moe_fusion_spec(experts) is spec


def test_transposed_view_reconstructs_checkpoint_weight():
    model = _Model()
    experts = model.layers[0].experts
    register_moe_fusion_spec(experts, _standard_spec(model, checkpoint_transposed=True))
    gate_up = torch.stack(
        [torch.cat([experts[i].gate_proj.weight, experts[i].up_proj.weight], dim=0) for i in range(2)], dim=0
    )

    views = {view.checkpoint_name: view for view in iter_moe_fusion_views(model)}

    assert torch.equal(views["layers.0.experts.gate_up_proj"].tensor_fn(), gate_up.transpose(1, 2))


def test_standard_spec_supports_custom_weight_concat_dimension():
    model = _Model()
    spec = build_standard_moe_fusion_spec(
        detected_projections={"gate_up_proj": {"split_into": ["gate_proj", "up_proj"], "concat_dim": 0}},
        num_experts=2,
        checkpoint_transposed=True,
        module=model.layers[0].experts,
    )

    assert spec.projections[0].concat_dim == 0


def test_bias_views_reconstruct_checkpoint_without_weight_transpose():
    model = _Model(bias=True)
    experts = model.layers[0].experts
    spec = MoEFusionSpec(
        num_experts=2,
        projections=(
            ProjectionFusionSpec(
                checkpoint_projection="gate_up_proj",
                source_projections=("gate_proj", "up_proj"),
                concat_dim=1,
                checkpoint_transposed=True,
                checkpoint_bias="gate_up_proj_bias",
            ),
            ProjectionFusionSpec(
                checkpoint_projection="down_proj",
                source_projections=("down_proj",),
                concat_dim=None,
                checkpoint_transposed=True,
                checkpoint_bias="down_proj_bias",
            ),
        ),
    )
    register_moe_fusion_spec(experts, spec)
    gate_up_bias = torch.stack(
        [torch.cat([experts[i].gate_proj.bias, experts[i].up_proj.bias], dim=0) for i in range(2)], dim=0
    )
    down_bias = torch.stack([experts[i].down_proj.bias for i in range(2)], dim=0)

    views = {view.checkpoint_name: view for view in iter_moe_fusion_views(model)}

    assert torch.equal(views["layers.0.experts.gate_up_proj_bias"].tensor_fn(), gate_up_bias)
    assert torch.equal(views["layers.0.experts.down_proj_bias"].tensor_fn(), down_bias)
    assert all(source.parameter == "bias" for source in views["layers.0.experts.gate_up_proj_bias"].sources)


def test_missing_source_has_contextual_error():
    model = _Model()
    experts = model.layers[0].experts
    register_moe_fusion_spec(experts, _standard_spec(model))
    del experts[1].up_proj

    views = {view.checkpoint_name: view for view in iter_moe_fusion_views(model)}

    with pytest.raises(ValueError, match=r"layers\.0\.experts.*up_proj.*expert 1"):
        views["layers.0.experts.gate_up_proj"].tensor_fn()


def test_marked_linearized_replacement_without_fusion_spec_is_rejected():
    model = nn.Module()
    model.block = nn.Module()
    model.block._auto_round_replaced_fused_moe = True

    with pytest.raises(ValueError, match=r"^block: linearized fused MoE replacement has no MoEFusionSpec$"):
        list(iter_moe_fusion_views(model))


def test_qwen3_5_replacement_registers_descendant_fusion_spec():
    from auto_round.modeling.fused_moe.qwen3_5_moe import LinearQwen3_5MoeSparseMoeBlock

    hidden_size = 3
    intermediate_size = 2
    num_experts = 2

    class TinyExperts(nn.Module):
        def __init__(self):
            super().__init__()
            self.gate_up_proj = nn.Parameter(torch.zeros(num_experts, 2 * intermediate_size, hidden_size))
            self.down_proj = nn.Parameter(torch.zeros(num_experts, hidden_size, intermediate_size))

    class TinyOriginal(nn.Module):
        def __init__(self):
            super().__init__()
            self.gate = nn.Identity()
            self.shared_expert = nn.Identity()
            self.experts = TinyExperts()
            self.shared_expert_gate = nn.Identity()

    text_config = type(
        "TextConfig",
        (),
        {
            "hidden_size": hidden_size,
            "hidden_act": "silu",
            "moe_intermediate_size": intermediate_size,
            "num_experts": num_experts,
        },
    )()
    config = type("Config", (), {"get_text_config": lambda self: text_config})()

    replacement = LinearQwen3_5MoeSparseMoeBlock(TinyOriginal(), config)

    assert replacement._auto_round_replaced_fused_moe is True
    assert get_moe_fusion_spec(replacement.experts) is not None
    assert [view.checkpoint_name for view in iter_moe_fusion_views(replacement)] == [
        "experts.gate_up_proj",
        "experts.down_proj",
    ]