Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

14.5 KB
Raw Blame
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Test script for moe_experts_interface.py - linear_loop experts implementation.

This verifies that:
1. linear_loop is registered with transformers 'ALL_EXPERTS_FUNCTIONS'
2. Fused expert weights are correctly unfused to nn.Linear layers
3. The forward pass produces correct results
"""

import pytest
import torch
from torch import nn

from auto_round.modeling.fused_moe.replace_modules import (
    ReplacementModuleBase,
    _apply_custom_replacements,
)
from auto_round.modeling.fused_moe.utils import (
    get_num_experts,
    is_linearized_layout,
)


def _skip_if_no_linear_loop():
    from auto_round.modeling.fused_moe.moe_experts_interface import is_linear_loop_available

    if not is_linear_loop_available():
        pytest.skip("transformers MOE integration not available")


def test_linear_loop_registration():
    """Test that linear_loop is registered with transformers."""
    from auto_round.modeling.fused_moe.moe_experts_interface import (
        register_linear_loop_experts,
    )

    _skip_if_no_linear_loop()
    success = register_linear_loop_experts()
    assert success, "Failed to register linear_loop"

    from transformers.integrations.moe import ALL_EXPERTS_FUNCTIONS

    assert "linear_loop" in ALL_EXPERTS_FUNCTIONS._global_mapping


def test_unfuse_experts_weights():
    """Test unfusing fused expert weights to nn.Linear layers."""
    from auto_round.modeling.fused_moe.fusion_spec import get_moe_fusion_spec
    from auto_round.modeling.fused_moe.moe_experts_interface import _unfuse_experts_weights_inplace

    # Create a mock fused experts module (Mixtral style - not transposed)
    num_experts = 4
    hidden_dim = 64
    intermediate_dim = 128

    class MockFusedExperts(nn.Module):
        def __init__(self):
            super().__init__()
            # Not transposed: (num_experts, 2*intermediate, hidden)
            self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))
            # (num_experts, hidden, intermediate)
            self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))
            self.act_fn = nn.SiLU()
            self.num_experts = num_experts
            self.has_bias = False
            self.is_transposed = False

    module = MockFusedExperts()

    # Store original weights for comparison
    original_gate_up = module.gate_up_proj.data.clone()
    original_down = module.down_proj.data.clone()

    # Unfuse (check_decorator=False since mock module doesn't have the decorator)
    success = _unfuse_experts_weights_inplace(module, check_decorator=False)
    assert success, "Failed to unfuse weights"

    spec = get_moe_fusion_spec(module)
    assert spec is not None
    assert spec.num_experts == num_experts
    assert [item.checkpoint_projection for item in spec.projections] == ["gate_up_proj", "down_proj"]
    assert spec.projections[0].source_projections == ("gate_proj", "up_proj")
    assert spec.projections[0].checkpoint_transposed is False

    # Verify structure - per-expert containers with individual Linear layers
    for i in range(num_experts):
        expert = getattr(module, str(i))
        assert isinstance(expert.gate_proj, nn.Linear), f"Expert {i} gate_proj is not nn.Linear"
        assert isinstance(expert.up_proj, nn.Linear), f"Expert {i} up_proj is not nn.Linear"
        assert isinstance(expert.down_proj, nn.Linear), f"Expert {i} down_proj is not nn.Linear"

    # Verify weights are preserved
    # gate_up_proj was (num_experts, 2*intermediate, hidden), not transposed
    # Split along dim 1: gate -> [:intermediate_dim], up -> [intermediate_dim:]
    for i in range(num_experts):
        expert = getattr(module, str(i))
        assert torch.allclose(
            expert.gate_proj.weight.data, original_gate_up[i, :intermediate_dim, :], atol=1e-6
        ), f"gate weight mismatch for expert {i}"
        assert torch.allclose(
            expert.up_proj.weight.data, original_gate_up[i, intermediate_dim:, :], atol=1e-6
        ), f"up weight mismatch for expert {i}"
        assert torch.allclose(
            expert.down_proj.weight.data, original_down[i], atol=1e-6
        ), f"down weight mismatch for expert {i}"


def test_unfuse_experts_weights_transposed():
    """Test unfusing transposed expert weights (Llama4/GptOss style)."""
    from auto_round.modeling.fused_moe.fusion_spec import get_moe_fusion_spec
    from auto_round.modeling.fused_moe.moe_experts_interface import _unfuse_experts_weights_inplace

    num_experts = 4
    hidden_dim = 64
    intermediate_dim = 128

    class MockFusedExpertsTransposed(nn.Module):
        def __init__(self):
            super().__init__()
            # Transposed: (num_experts, hidden, 2*intermediate)
            self.gate_up_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, 2 * intermediate_dim))
            # Transposed: (num_experts, intermediate, hidden)
            self.down_proj = nn.Parameter(torch.randn(num_experts, intermediate_dim, hidden_dim))
            self.act_fn = nn.SiLU()
            self.num_experts = num_experts
            self.is_transposed = True
            self.has_bias = False

    module = MockFusedExpertsTransposed()

    # Store original weights for comparison
    original_gate_up = module.gate_up_proj.data.clone()
    original_down = module.down_proj.data.clone()

    # Unfuse (check_decorator=False since mock module doesn't have the decorator)
    success = _unfuse_experts_weights_inplace(module, check_decorator=False)
    assert success, "Failed to unfuse transposed weights"

    spec = get_moe_fusion_spec(module)
    assert spec is not None
    assert spec.num_experts == num_experts
    assert [item.checkpoint_projection for item in spec.projections] == ["gate_up_proj", "down_proj"]
    assert spec.projections[0].source_projections == ("gate_proj", "up_proj")
    assert spec.projections[0].checkpoint_transposed is True

    # Verify structure - per-expert containers with individual Linear layers
    for i in range(num_experts):
        expert = getattr(module, str(i))
        assert isinstance(expert.gate_proj, nn.Linear), f"Expert {i} gate_proj is not nn.Linear"
        assert isinstance(expert.up_proj, nn.Linear), f"Expert {i} up_proj is not nn.Linear"
        assert isinstance(expert.down_proj, nn.Linear), f"Expert {i} down_proj is not nn.Linear"

    # Verify weights are correctly transposed
    # gate_up_proj was (num_experts, hidden, 2*intermediate), transposed
    # Split along dim 2: gate -> [:, :intermediate_dim], up -> [:, intermediate_dim:]
    # Then transposed: (hidden, intermediate) -> (intermediate, hidden)
    for i in range(num_experts):
        expert = getattr(module, str(i))
        assert torch.allclose(
            expert.gate_proj.weight.data, original_gate_up[i, :, :intermediate_dim].t(), atol=1e-6
        ), f"gate weight mismatch for expert {i}"
        assert torch.allclose(
            expert.up_proj.weight.data, original_gate_up[i, :, intermediate_dim:].t(), atol=1e-6
        ), f"up weight mismatch for expert {i}"
        assert torch.allclose(
            expert.down_proj.weight.data, original_down[i].t(), atol=1e-6
        ), f"down weight mismatch for expert {i}"


def test_linear_loop_forward():
    """Test that linear_loop forward produces correct results."""
    from auto_round.modeling.fused_moe.moe_experts_interface import (
        _unfuse_experts_weights_inplace,
        linear_loop_experts_forward,
    )

    num_experts = 4
    hidden_dim = 64
    intermediate_dim = 128
    num_tokens = 10
    top_k = 2

    # Create module with unfused weights
    class MockExperts(nn.Module):
        def __init__(self):
            super().__init__()
            self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))
            self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))
            self.act_fn = nn.SiLU()
            self.num_experts = num_experts
            self.has_bias = False
            self.is_transposed = False

    module = MockExperts()

    # Unfuse weights (check_decorator=False since mock module doesn't have the decorator)
    _unfuse_experts_weights_inplace(module, check_decorator=False)

    # Create inputs
    hidden_states = torch.randn(num_tokens, hidden_dim)
    top_k_index = torch.randint(0, num_experts, (num_tokens, top_k))
    top_k_weights = torch.softmax(torch.randn(num_tokens, top_k), dim=-1)

    # Run forward
    output = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)

    # Verify output shape
    assert output.shape == hidden_states.shape, f"Output shape mismatch: {output.shape} vs {hidden_states.shape}"

    # Verify output is not all zeros (sanity check)
    assert not torch.allclose(output, torch.zeros_like(output)), "Output is all zeros"


def test_linear_loop_force_routing_preserves_original_output(monkeypatch):
    """Forced expert coverage should not change the returned MoE output."""
    from auto_round.modeling.fused_moe.moe_experts_interface import (
        _unfuse_experts_weights_inplace,
        linear_loop_experts_forward,
    )

    num_experts = 4
    hidden_dim = 16
    intermediate_dim = 32
    num_tokens = 9
    top_k = 2

    class MockExperts(nn.Module):
        def __init__(self):
            super().__init__()
            self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))
            self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))
            self.act_fn = nn.SiLU()
            self.num_experts = num_experts
            self.has_bias = False
            self.is_transposed = False

    torch.manual_seed(7)
    module = MockExperts()
    _unfuse_experts_weights_inplace(module, check_decorator=False)

    hidden_states = torch.randn(num_tokens, hidden_dim)
    top_k_index = torch.randint(0, num_experts, (num_tokens, top_k))
    top_k_weights = torch.softmax(torch.randn(num_tokens, top_k), dim=-1)

    monkeypatch.setenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "0")
    output_normal = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)

    monkeypatch.setenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "1")
    output_forced = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)

    assert torch.allclose(output_normal, output_forced, atol=1e-6, rtol=1e-5)
    assert hasattr(module, "_auto_round_forced_route_offset")


def test_prepare_model_for_moe_quantization():
    """Test the full prepare_model_for_moe_quantization flow."""
    from auto_round.modeling.fused_moe.moe_experts_interface import (
        prepare_model_for_moe_quantization,
        resolve_experts_implementation,
    )

    _skip_if_no_linear_loop()

    num_experts = 4
    hidden_dim = 64
    intermediate_dim = 128

    # Create a mock model with fused experts
    class MockConfig:
        def __init__(self):
            self._experts_implementation = "eager"

    class MockExpertsModule(nn.Module):
        def __init__(self):
            super().__init__()
            self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))
            self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))
            self.act_fn = nn.SiLU()
            self.num_experts = num_experts
            self.has_bias = False
            self.is_transposed = False

        def forward(self, x):
            pass

    # Add __wrapped__ to simulate @use_experts_implementation decorator
    MockExpertsModule.forward.__wrapped__ = True

    class MockModel(nn.Module):
        def __init__(self):
            super().__init__()
            self.config = MockConfig()
            self.layer = nn.ModuleDict({"experts": MockExpertsModule()})

    model = MockModel()

    # Prepare for quantization
    unfused = prepare_model_for_moe_quantization(model)

    # Verify
    assert model.config._experts_implementation == resolve_experts_implementation()
    assert len(unfused) == 1
    experts = model.layer["experts"]
    expert_0 = getattr(experts, "0")
    assert isinstance(expert_0.gate_proj, nn.Linear)


class _TinyExpert(nn.Module):
    def __init__(self):
        super().__init__()
        self.gate_proj = nn.Linear(4, 8, bias=False)
        self.up_proj = nn.Linear(4, 8, bias=False)
        self.down_proj = nn.Linear(8, 4, bias=False)


def _build_linearized_experts(num_experts: int, with_act_fn: bool = True) -> nn.ModuleList:
    experts = nn.ModuleList([_TinyExpert() for _ in range(num_experts)])
    if with_act_fn:
        experts.act_fn = nn.SiLU()
    return experts


def test_utils_get_num_experts_ignores_non_numeric_modules_in_linearized_layout():
    experts = _build_linearized_experts(num_experts=3, with_act_fn=True)

    assert is_linearized_layout(experts)
    assert len(experts) == 4  # 3 experts + act_fn module
    assert get_num_experts(experts) == 3


def test_apply_custom_replacements_skips_linearized_moe_and_logs_reason(monkeypatch):
    messages = []

    # Capture info logs to assert the skip reason is explicit.
    from auto_round.modeling.fused_moe import replace_modules as rm

    monkeypatch.setattr(rm.logger, "info", lambda msg: messages.append(str(msg)))

    class DummyMoEForLinearizeSkipUT(nn.Module):
        def __init__(self):
            super().__init__()
            self.experts = _build_linearized_experts(num_experts=2, with_act_fn=True)

    class DummyReplacementForLinearizeSkipUT(ReplacementModuleBase):
        @classmethod
        def original_module_class(cls) -> str:
            return "DummyMoEForLinearizeSkipUT"

        @classmethod
        def from_original(cls, original: nn.Module, config):
            return cls(original)

        def __init__(self, original: nn.Module):
            super().__init__(original)

    model = nn.Module()
    model.config = type("Cfg", (), {"model_type": "unit_test"})()
    model.block = DummyMoEForLinearizeSkipUT()

    replaced = _apply_custom_replacements(model)

    assert replaced == []
    assert isinstance(model.block, DummyMoEForLinearizeSkipUT)
    assert any("skipped because linearize_moe" in msg for msg in messages)