PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2026 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Test script for moe_experts_interface.py - linear_loop experts implementation.
This verifies that:
1. linear_loop is registered with transformers 'ALL_EXPERTS_FUNCTIONS'
2. Fused expert weights are correctly unfused to nn.Linear layers
3. The forward pass produces correct results
"""
import pytest
import torch
from torch import nn
from auto_round.modeling.fused_moe.replace_modules import (
ReplacementModuleBase,
_apply_custom_replacements,
)
from auto_round.modeling.fused_moe.utils import (
get_num_experts,
is_linearized_layout,
)
def _skip_if_no_linear_loop():
from auto_round.modeling.fused_moe.moe_experts_interface import is_linear_loop_available
if not is_linear_loop_available():
pytest.skip("transformers MOE integration not available")
def test_linear_loop_registration():
"""Test that linear_loop is registered with transformers."""
from auto_round.modeling.fused_moe.moe_experts_interface import (
register_linear_loop_experts,
)
_skip_if_no_linear_loop()
success = register_linear_loop_experts()
assert success, "Failed to register linear_loop"
from transformers.integrations.moe import ALL_EXPERTS_FUNCTIONS
assert "linear_loop" in ALL_EXPERTS_FUNCTIONS._global_mapping
def test_unfuse_experts_weights():
"""Test unfusing fused expert weights to nn.Linear layers."""
from auto_round.modeling.fused_moe.fusion_spec import get_moe_fusion_spec
from auto_round.modeling.fused_moe.moe_experts_interface import _unfuse_experts_weights_inplace
# Create a mock fused experts module (Mixtral style - not transposed)
num_experts = 4
hidden_dim = 64
intermediate_dim = 128
class MockFusedExperts(nn.Module):
def __init__(self):
super().__init__()
# Not transposed: (num_experts, 2*intermediate, hidden)
self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))
# (num_experts, hidden, intermediate)
self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))
self.act_fn = nn.SiLU()
self.num_experts = num_experts
self.has_bias = False
self.is_transposed = False
module = MockFusedExperts()
# Store original weights for comparison
original_gate_up = module.gate_up_proj.data.clone()
original_down = module.down_proj.data.clone()
# Unfuse (check_decorator=False since mock module doesn't have the decorator)
success = _unfuse_experts_weights_inplace(module, check_decorator=False)
assert success, "Failed to unfuse weights"
spec = get_moe_fusion_spec(module)
assert spec is not None
assert spec.num_experts == num_experts
assert [item.checkpoint_projection for item in spec.projections] == ["gate_up_proj", "down_proj"]
assert spec.projections[0].source_projections == ("gate_proj", "up_proj")
assert spec.projections[0].checkpoint_transposed is False
# Verify structure - per-expert containers with individual Linear layers
for i in range(num_experts):
expert = getattr(module, str(i))
assert isinstance(expert.gate_proj, nn.Linear), f"Expert {i} gate_proj is not nn.Linear"
assert isinstance(expert.up_proj, nn.Linear), f"Expert {i} up_proj is not nn.Linear"
assert isinstance(expert.down_proj, nn.Linear), f"Expert {i} down_proj is not nn.Linear"
# Verify weights are preserved
# gate_up_proj was (num_experts, 2*intermediate, hidden), not transposed
# Split along dim 1: gate -> [:intermediate_dim], up -> [intermediate_dim:]
for i in range(num_experts):
expert = getattr(module, str(i))
assert torch.allclose(
expert.gate_proj.weight.data, original_gate_up[i, :intermediate_dim, :], atol=1e-6
), f"gate weight mismatch for expert {i}"
assert torch.allclose(
expert.up_proj.weight.data, original_gate_up[i, intermediate_dim:, :], atol=1e-6
), f"up weight mismatch for expert {i}"
assert torch.allclose(
expert.down_proj.weight.data, original_down[i], atol=1e-6
), f"down weight mismatch for expert {i}"
def test_unfuse_experts_weights_transposed():
"""Test unfusing transposed expert weights (Llama4/GptOss style)."""
from auto_round.modeling.fused_moe.fusion_spec import get_moe_fusion_spec
from auto_round.modeling.fused_moe.moe_experts_interface import _unfuse_experts_weights_inplace
num_experts = 4
hidden_dim = 64
intermediate_dim = 128
class MockFusedExpertsTransposed(nn.Module):
def __init__(self):
super().__init__()
# Transposed: (num_experts, hidden, 2*intermediate)
self.gate_up_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, 2 * intermediate_dim))
# Transposed: (num_experts, intermediate, hidden)
self.down_proj = nn.Parameter(torch.randn(num_experts, intermediate_dim, hidden_dim))
self.act_fn = nn.SiLU()
self.num_experts = num_experts
self.is_transposed = True
self.has_bias = False
module = MockFusedExpertsTransposed()
# Store original weights for comparison
original_gate_up = module.gate_up_proj.data.clone()
original_down = module.down_proj.data.clone()
# Unfuse (check_decorator=False since mock module doesn't have the decorator)
success = _unfuse_experts_weights_inplace(module, check_decorator=False)
assert success, "Failed to unfuse transposed weights"
spec = get_moe_fusion_spec(module)
assert spec is not None
assert spec.num_experts == num_experts
assert [item.checkpoint_projection for item in spec.projections] == ["gate_up_proj", "down_proj"]
assert spec.projections[0].source_projections == ("gate_proj", "up_proj")
assert spec.projections[0].checkpoint_transposed is True
# Verify structure - per-expert containers with individual Linear layers
for i in range(num_experts):
expert = getattr(module, str(i))
assert isinstance(expert.gate_proj, nn.Linear), f"Expert {i} gate_proj is not nn.Linear"
assert isinstance(expert.up_proj, nn.Linear), f"Expert {i} up_proj is not nn.Linear"
assert isinstance(expert.down_proj, nn.Linear), f"Expert {i} down_proj is not nn.Linear"
# Verify weights are correctly transposed
# gate_up_proj was (num_experts, hidden, 2*intermediate), transposed
# Split along dim 2: gate -> [:, :intermediate_dim], up -> [:, intermediate_dim:]
# Then transposed: (hidden, intermediate) -> (intermediate, hidden)
for i in range(num_experts):
expert = getattr(module, str(i))
assert torch.allclose(
expert.gate_proj.weight.data, original_gate_up[i, :, :intermediate_dim].t(), atol=1e-6
), f"gate weight mismatch for expert {i}"
assert torch.allclose(
expert.up_proj.weight.data, original_gate_up[i, :, intermediate_dim:].t(), atol=1e-6
), f"up weight mismatch for expert {i}"
assert torch.allclose(
expert.down_proj.weight.data, original_down[i].t(), atol=1e-6
), f"down weight mismatch for expert {i}"
def test_linear_loop_forward():
"""Test that linear_loop forward produces correct results."""
from auto_round.modeling.fused_moe.moe_experts_interface import (
_unfuse_experts_weights_inplace,
linear_loop_experts_forward,
)
num_experts = 4
hidden_dim = 64
intermediate_dim = 128
num_tokens = 10
top_k = 2
# Create module with unfused weights
class MockExperts(nn.Module):
def __init__(self):
super().__init__()
self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))
self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))
self.act_fn = nn.SiLU()
self.num_experts = num_experts
self.has_bias = False
self.is_transposed = False
module = MockExperts()
# Unfuse weights (check_decorator=False since mock module doesn't have the decorator)
_unfuse_experts_weights_inplace(module, check_decorator=False)
# Create inputs
hidden_states = torch.randn(num_tokens, hidden_dim)
top_k_index = torch.randint(0, num_experts, (num_tokens, top_k))
top_k_weights = torch.softmax(torch.randn(num_tokens, top_k), dim=-1)
# Run forward
output = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)
# Verify output shape
assert output.shape == hidden_states.shape, f"Output shape mismatch: {output.shape} vs {hidden_states.shape}"
# Verify output is not all zeros (sanity check)
assert not torch.allclose(output, torch.zeros_like(output)), "Output is all zeros"
def test_linear_loop_force_routing_preserves_original_output(monkeypatch):
"""Forced expert coverage should not change the returned MoE output."""
from auto_round.modeling.fused_moe.moe_experts_interface import (
_unfuse_experts_weights_inplace,
linear_loop_experts_forward,
)
num_experts = 4
hidden_dim = 16
intermediate_dim = 32
num_tokens = 9
top_k = 2
class MockExperts(nn.Module):
def __init__(self):
super().__init__()
self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))
self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))
self.act_fn = nn.SiLU()
self.num_experts = num_experts
self.has_bias = False
self.is_transposed = False
torch.manual_seed(7)
module = MockExperts()
_unfuse_experts_weights_inplace(module, check_decorator=False)
hidden_states = torch.randn(num_tokens, hidden_dim)
top_k_index = torch.randint(0, num_experts, (num_tokens, top_k))
top_k_weights = torch.softmax(torch.randn(num_tokens, top_k), dim=-1)
monkeypatch.setenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "0")
output_normal = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)
monkeypatch.setenv("AR_FORCE_MOE_ROUTING_ALL_EXPERTS", "1")
output_forced = linear_loop_experts_forward(module, hidden_states, top_k_index, top_k_weights)
assert torch.allclose(output_normal, output_forced, atol=1e-6, rtol=1e-5)
assert hasattr(module, "_auto_round_forced_route_offset")
def test_prepare_model_for_moe_quantization():
"""Test the full prepare_model_for_moe_quantization flow."""
from auto_round.modeling.fused_moe.moe_experts_interface import (
prepare_model_for_moe_quantization,
resolve_experts_implementation,
)
_skip_if_no_linear_loop()
num_experts = 4
hidden_dim = 64
intermediate_dim = 128
# Create a mock model with fused experts
class MockConfig:
def __init__(self):
self._experts_implementation = "eager"
class MockExpertsModule(nn.Module):
def __init__(self):
super().__init__()
self.gate_up_proj = nn.Parameter(torch.randn(num_experts, 2 * intermediate_dim, hidden_dim))
self.down_proj = nn.Parameter(torch.randn(num_experts, hidden_dim, intermediate_dim))
self.act_fn = nn.SiLU()
self.num_experts = num_experts
self.has_bias = False
self.is_transposed = False
def forward(self, x):
pass
# Add __wrapped__ to simulate @use_experts_implementation decorator
MockExpertsModule.forward.__wrapped__ = True
class MockModel(nn.Module):
def __init__(self):
super().__init__()
self.config = MockConfig()
self.layer = nn.ModuleDict({"experts": MockExpertsModule()})
model = MockModel()
# Prepare for quantization
unfused = prepare_model_for_moe_quantization(model)
# Verify
assert model.config._experts_implementation == resolve_experts_implementation()
assert len(unfused) == 1
experts = model.layer["experts"]
expert_0 = getattr(experts, "0")
assert isinstance(expert_0.gate_proj, nn.Linear)
class _TinyExpert(nn.Module):
def __init__(self):
super().__init__()
self.gate_proj = nn.Linear(4, 8, bias=False)
self.up_proj = nn.Linear(4, 8, bias=False)
self.down_proj = nn.Linear(8, 4, bias=False)
def _build_linearized_experts(num_experts: int, with_act_fn: bool = True) -> nn.ModuleList:
experts = nn.ModuleList([_TinyExpert() for _ in range(num_experts)])
if with_act_fn:
experts.act_fn = nn.SiLU()
return experts
def test_utils_get_num_experts_ignores_non_numeric_modules_in_linearized_layout():
experts = _build_linearized_experts(num_experts=3, with_act_fn=True)
assert is_linearized_layout(experts)
assert len(experts) == 4 # 3 experts + act_fn module
assert get_num_experts(experts) == 3
def test_apply_custom_replacements_skips_linearized_moe_and_logs_reason(monkeypatch):
messages = []
# Capture info logs to assert the skip reason is explicit.
from auto_round.modeling.fused_moe import replace_modules as rm
monkeypatch.setattr(rm.logger, "info", lambda msg: messages.append(str(msg)))
class DummyMoEForLinearizeSkipUT(nn.Module):
def __init__(self):
super().__init__()
self.experts = _build_linearized_experts(num_experts=2, with_act_fn=True)
class DummyReplacementForLinearizeSkipUT(ReplacementModuleBase):
@classmethod
def original_module_class(cls) -> str:
return "DummyMoEForLinearizeSkipUT"
@classmethod
def from_original(cls, original: nn.Module, config):
return cls(original)
def __init__(self, original: nn.Module):
super().__init__(original)
model = nn.Module()
model.config = type("Cfg", (), {"model_type": "unit_test"})()
model.block = DummyMoEForLinearizeSkipUT()
replaced = _apply_custom_replacements(model)
assert replaced == []
assert isinstance(model.block, DummyMoEForLinearizeSkipUT)
assert any("skipped because linearize_moe" in msg for msg in messages)