PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright 2025 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
"""Tests for the per-device grouped experts plans and the align-hook exemption."""
import unittest
import torch
from torch import nn
from auto_round.modeling.fused_moe.grouped_experts import (
_hooks_are_alignment_only,
_projection_is_supported,
_run_routes,
)
class TestHookWhitelistResolution(unittest.TestCase):
"""Missing accelerate hook classes are probed once per process, not per call."""
def test_missing_classes_log_once(self):
import sys
import types
from unittest import mock
from auto_round.modeling.fused_moe import grouped_experts
orig = sys.modules.get("accelerate.hooks")
stub = types.ModuleType("accelerate.hooks") # carries neither hook class
sys.modules["accelerate.hooks"] = stub
try:
grouped_experts._ACCELERATE_HOOK_WHITELIST = None
with mock.patch.object(grouped_experts, "logger") as spy:
self.assertEqual(grouped_experts._accelerate_hook_whitelist(), ())
notices = spy.warning.call_count + spy.debug.call_count
self.assertEqual(notices, 2)
# the cached resolution must not repeat the notices
self.assertEqual(grouped_experts._accelerate_hook_whitelist(), ())
self.assertEqual(spy.warning.call_count + spy.debug.call_count, notices)
lin = _linear()
lin.register_forward_hook(lambda *a: None) # any hook reaches the whitelist gate
self.assertFalse(_hooks_are_alignment_only(lin))
finally:
if orig is not None:
sys.modules["accelerate.hooks"] = orig
grouped_experts._ACCELERATE_HOOK_WHITELIST = None
def _linear(out=4, inp=4):
lin = nn.Linear(inp, out, bias=False)
with torch.no_grad():
lin.weight.copy_(torch.randn_like(lin.weight))
return lin
def _expert(gate=True, seed=0):
torch.manual_seed(seed)
mod = nn.Module()
mod.up_proj = _linear()
mod.down_proj = _linear()
if gate:
mod.gate_proj = _linear()
return mod
class _Experts(nn.Module):
def __init__(self, n=2):
super().__init__()
self.n = n
for i in range(n):
setattr(self, str(i), _expert(seed=i))
self.act_fn = nn.functional.silu
def forward(self, x, idx, w):
return _run_routes(self, x, idx, w, self.n)
class TestHookExemption(unittest.TestCase):
def test_no_hooks_is_alignment_only(self):
self.assertTrue(_hooks_are_alignment_only(_linear()))
def test_plain_hook_is_not_exempt(self):
lin = _linear()
lin.register_forward_hook(lambda m, i, o: o)
self.assertFalse(_hooks_are_alignment_only(lin))
self.assertFalse(_projection_is_supported(lin)) # calibration hooks must fire
def test_align_hook_without_offload_is_exempt(self):
from accelerate.hooks import AlignDevicesHook
lin = _linear()
lin.register_forward_pre_hook(AlignDevicesHook(execution_device=lin.weight.device))
self.assertTrue(_hooks_are_alignment_only(lin))
self.assertTrue(_projection_is_supported(lin))
def test_align_hook_with_offload_is_not_exempt(self):
from accelerate.hooks import AlignDevicesHook
lin = _linear()
hook = AlignDevicesHook(execution_device=lin.weight.device, offload=True)
lin.register_forward_pre_hook(hook)
self.assertFalse(_hooks_are_alignment_only(lin))
self.assertFalse(_projection_is_supported(lin))
class TestRunRoutesSingleGroupCPU(unittest.TestCase):
def test_matches_reference(self):
torch.manual_seed(7)
experts = _Experts(4)
n_tokens, n_experts, top_k, hidden = 12, 4, 2, 4
x = torch.randn(n_tokens, hidden)
idx = torch.randint(0, n_experts, (n_tokens, top_k))
w = torch.rand(n_tokens, top_k)
out = experts(x, idx, w)
ref = torch.zeros_like(x)
for t in range(n_tokens):
for k in range(top_k):
e = getattr(experts, str(int(idx[t, k])))
h = nn.functional.silu(e.gate_proj(x[t])) * e.up_proj(x[t])
ref[t] += w[t, k] * e.down_proj(h)
torch.testing.assert_close(out, ref, rtol=1e-5, atol=1e-5)
def _wrapped_linear(self, bits, seed):
from auto_round.wrapper import WrapperLinear
layer = _linear()
torch.manual_seed(seed)
with torch.no_grad():
layer.weight.copy_(torch.randn_like(layer.weight))
layer.bits = bits
layer.group_size = -1
layer.sym = True
layer.data_type = "int"
layer.iters = 0
layer.act_bits = 16
layer.scale_dtype = torch.float16
return WrapperLinear(
layer,
device="cpu",
enable_minmax_tuning=False,
enable_norm_bias_tuning=False,
enable_round_tuning=False,
enable_torch_compile=False,
disable_opt_rtn=False,
iters=0,
)
def test_mixed_bits_across_experts_falls_back(self):
# different quant signatures inside one device group must reject the
# grouped path (sharing one grouped GEMM across differently-quantized
# experts is unsound), not silently merge them. Plain Linears carry no
# quant signature at all (None == None), so the mix needs wrappers.
experts = _Experts(2)
getattr(experts, "0").up_proj = self._wrapped_linear(bits=4, seed=1)
getattr(experts, "1").up_proj = self._wrapped_linear(bits=8, seed=2)
x = torch.randn(6, 4)
idx = torch.tensor([[0, 1]] * 6) # both experts active
w = torch.rand(6, 2)
self.assertIsNone(_run_routes(experts, x, idx, w, experts.n))
def test_plain_before_wrapper_falls_back(self):
# a plain Linear's quantization signature is None -- a VALID value that
# must not double as the unset marker. Seeding the reference from the
# first position rejects the mix regardless of expert ordering; the
# old sentinel latched the wrapper's signature without comparison when
# the plain expert came first and accepted the mixed group.
x = torch.randn(6, 4)
idx = torch.tensor([[0, 1]] * 6)
w = torch.rand(6, 2)
experts = _Experts(2)
getattr(experts, "1").up_proj = self._wrapped_linear(bits=4, seed=3) # plain first
self.assertIsNone(_run_routes(experts, x, idx, w, experts.n))
experts = _Experts(2)
getattr(experts, "0").up_proj = self._wrapped_linear(bits=4, seed=4) # wrapper first
self.assertIsNone(_run_routes(experts, x, idx, w, experts.n))
def test_weightless_packed_linear_falls_back(self):
# packed QuantLinear types (e.g. MXFP4QuantLinear) store packed weights
# and expose no .weight; the support probe must reject them BEFORE any
# device read instead of raising AttributeError (CI: qwen3-vl-moe-mxfp)
experts = _Experts(2)
getattr(experts, "0").up_proj = nn.Module() # no .weight, not a wrapper
x = torch.randn(6, 4)
idx = torch.tensor([[0, 1]] * 6)
w = torch.rand(6, 2)
self.assertIsNone(_run_routes(experts, x, idx, w, experts.n))
def test_missing_slot_falls_back(self):
torch.manual_seed(3)
experts = _Experts(2)
object.__setattr__(getattr(experts, "1"), "up_proj", None) # slot absent on expert 1
x = torch.randn(6, 4)
idx = torch.tensor([[0, 1]] * 6)
w = torch.rand(6, 2)
self.assertIsNone(_run_routes(experts, x, idx, w, experts.n))
@unittest.skipIf(
not (torch.cuda.is_available() and torch.cuda.device_count() >= 3),
"needs >=3 CUDA devices (inputs live on a third device)",
)
class TestRunRoutesMultiDeviceCUDA(unittest.TestCase):
def test_two_device_groups_match_reference(self):
torch.manual_seed(11)
experts = _Experts(4) # tiny CPU build; nn.Module.to() cannot copy out of meta
# experts 0,1 on cuda:0; 2,3 on cuda:1 (weights move explicitly)
for i in range(4):
e = getattr(experts, str(i))
dev = f"cuda:{i // 2}"
for slot in ("up_proj", "down_proj", "gate_proj"):
getattr(e, slot).to(dev)
n_tokens, n_experts, top_k, hidden = 64, 4, 2, 4
x = torch.randn(n_tokens, hidden, device="cuda:2") # input on a third device
idx = torch.randint(0, n_experts, (n_tokens, top_k), device="cuda:2")
w = torch.rand(n_tokens, top_k, device="cuda:2")
out = experts(x, idx, w)
self.assertEqual(out.device, x.device)
# reference on cuda:2 by explicit per-expert moves
ref = torch.zeros_like(x)
for t in range(n_tokens):
for k in range(top_k):
e = getattr(experts, str(int(idx[t, k])))
dev = next(p.device for p in e.up_proj.parameters())
xt = x[t].to(dev)
h = nn.functional.silu(e.gate_proj(xt)) * e.up_proj(xt)
ref[t] += (w[t, k] * e.down_proj(h).to(x.device)).to(x.device)
torch.testing.assert_close(out, ref, rtol=1e-4, atol=1e-4)
class TestAtomicPlacement(unittest.TestCase):
"""The allocator must never split one expert's projections across devices."""
def _layer_dict(self, n_experts=6):
d = {}
for i in range(n_experts):
for slot in ("gate_proj", "up_proj", "down_proj"):
d[f"mlp.experts.{i}.{slot}"] = {"param_memory": 1.0, "is_moe_expert": True}
d["self_attn.q_proj"] = {"param_memory": 1.0, "is_moe_expert": False}
d["mlp.shared_experts.gate_proj"] = {"param_memory": 1.0, "is_moe_expert": False}
return d
def test_preassign_keeps_experts_atomic_and_spreads(self):
from auto_round.utils.device import _preassign_moe_experts
layer_dict = self._layer_dict(8)
budgets = {"cuda:0": 12.0, "cuda:1": 12.0}
assigned = _preassign_moe_experts(layer_dict, budgets, ["cuda:0", "cuda:1"], mem_per_param=1.0)
self.assertTrue(assigned)
# every expert's three slots share one device
for i in range(8):
devs = {assigned[f"mlp.experts.{i}.{s}"] for s in ("gate_proj", "up_proj", "down_proj")}
self.assertEqual(len(devs), 1, f"expert {i} split: {devs}")
# spread across both devices
self.assertEqual({d for d in assigned.values()}, {"cuda:0", "cuda:1"})
def test_preassign_shrinks_chunk_on_overflow(self):
from auto_round.utils.device import _preassign_moe_experts
# budgets so tight that a 16-expert chunk cannot fit, but single experts can
layer_dict = self._layer_dict(6)
budgets = {"cuda:0": 3.5, "cuda:1": 3.5}
assigned = _preassign_moe_experts(layer_dict, budgets, ["cuda:0", "cuda:1"], mem_per_param=1.0)
for i in range(6):
devs = {assigned.get(f"mlp.experts.{i}.{s}") for s in ("gate_proj", "up_proj", "down_proj")}
self.assertEqual(len(devs), 1, f"expert {i} split or unplaced: {devs}")
def test_balancer_groups_experts_atomically(self):
from auto_round.utils.device import _allocate_layers_to_devices
layer_dict = self._layer_dict(4)
budgets = {"cuda:0": 100.0, "cuda:1": 100.0}
device_map, _names = _allocate_layers_to_devices(layer_dict, budgets, ["cuda:0", "cuda:1"], 1.0)
for i in range(4):
devs = {device_map[f"mlp.experts.{i}.{s}"] for s in ("gate_proj", "up_proj", "down_proj")}
self.assertEqual(len(devs), 1, f"expert {i} split by balancer: {devs}")