Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support fused MoE quantization and export for GGUF format (#2072)

Heng Guo1fca0c748cfa
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index 1da140d04..8f6f3efd3 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -77,7 +77,7 @@ from auto_round.utils.device import (
patch_xpu_sdpa_drop_causal_mask,
set_non_auto_device_map,
)
-from auto_round.utils.device_manager import device_manager
+from auto_round.utils.device_manager import default_enable_torch_compile, device_manager
from auto_round.utils.offload import OffloadManager
@@ -354,13 +354,19 @@ class BaseOrchestrator(object):
self.nblocks = nblocks
if enable_torch_compile is None:
- enable_torch_compile = sys.platform != "win32"
+ enable_torch_compile = default_enable_torch_compile(self.device, platform_name=sys.platform)
if not enable_torch_compile:
- logger.warning_once(
- "`torch.compile` is disabled by default on Windows because TorchInductor requires the MSVC "
- "`cl.exe` compiler, which may not be available. Pass `enable_torch_compile=True` or use "
- "`--enable_torch_compile` to force enable it."
- )
+ if self.device == "xpu":
+ logger.warning_once(
+ "`torch.compile` is disabled by default on XPU for compatibility. "
+ "Pass `enable_torch_compile=True` or use `--enable_torch_compile` to force enable it."
+ )
+ else:
+ logger.warning_once(
+ "`torch.compile` is disabled by default on Windows because TorchInductor requires the MSVC "
+ "`cl.exe` compiler, which may not be available. Pass `enable_torch_compile=True` or use "
+ "`--enable_torch_compile` to force enable it."
+ )
elif enable_torch_compile and sys.platform == "win32":
logger.warning_once(
"Forcing `torch.compile` on Windows. TorchInductor may fail if the MSVC `cl.exe` compiler "
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index 3c59ba15b..b6ad9839b 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -110,6 +110,7 @@ from auto_round.logger import logger
from auto_round.schemes import PRESET_SCHEMES, QuantizationScheme, preset_name_to_scheme
from auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, compress_layer_names, to_standard_regex
from auto_round.utils.device import clear_memory, compile_func, memory_monitor
+from auto_round.utils.device_manager import default_enable_torch_compile
from auto_round.utils.missing_tensors import quantize_weight_rtn, split_fused_expert_tensors
# ---------------------------------------------------------------------------
@@ -1990,7 +1991,11 @@ class _ModelFreeCompressorCore:
self.device = device
self.quant_lm_head = quant_lm_head
self.quant_nontext_module = quant_nontext_module
- self.enable_torch_compile = sys.platform != "win32" if enable_torch_compile is None else enable_torch_compile
+ self.enable_torch_compile = (
+ default_enable_torch_compile(device, platform_name=sys.platform)
+ if enable_torch_compile is None
+ else enable_torch_compile
+ )
# --- derived state populated during run() ---
self.scheme_obj: QuantizationScheme | None = None
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 93c1b7e63..c922be9a5 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -48,6 +48,12 @@ from transformers import AutoConfig
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector
from auto_round.export.export_to_gguf.hf_checkpoint_restorer import HFCheckpointRestorer, RestoredTensor
+from auto_round.export.export_to_gguf.moe_adapter import (
+ pack_moe_output,
+ resolve_moe_output,
+ validate_moe_imatrices,
+ validate_moe_source_qtypes,
+)
from auto_round.export.export_to_gguf.packing import ggml_quant
from auto_round.utils import (
LazyImport,
@@ -192,6 +198,7 @@ def get_restored_tensors(cls) -> Iterator[RestoredTensor]:
lambda tensor=tensor: tensor,
restored.hf_names,
restored.transform_kind,
+ restored.moe_sources,
)
for name, tensor in _iter_extra_tensors(cls):
@@ -199,7 +206,13 @@ def get_restored_tensors(cls) -> Iterator[RestoredTensor]:
if pending is None:
yield RestoredTensor(name, lambda tensor=tensor: tensor, (name,), "extra")
else:
- yield RestoredTensor(name, lambda tensor=tensor: tensor, pending.hf_names, pending.transform_kind)
+ yield RestoredTensor(
+ name,
+ lambda tensor=tensor: tensor,
+ pending.hf_names,
+ pending.transform_kind,
+ pending.moe_sources,
+ )
# gguf's canonical (architecture-independent) tensor names for the three merged MoE FFN
@@ -459,14 +472,14 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d
module = get_module(cls.model, layer_name)
kwargs = {"scale": None, "zp": None, "d_scale": None, "d_wmin": None, "wmin": None, "imatrix": None}
source_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype, explicit_only=True)
- use_layer_attrs = use_layer_attrs and source_qtype == data_qtype
if use_layer_attrs:
+ compatible_stored_qtype = source_qtype == data_qtype
kwargs = {
- "scale": module.scale if hasattr(module, "scale") else None,
- "zp": module.zp if hasattr(module, "zp") else None,
- "d_scale": module.w_d_scale if hasattr(module, "w_d_scale") else None,
- "d_wmin": module.w_d_wmin if hasattr(module, "w_d_wmin") else None,
- "wmin": module.w_wmin if hasattr(module, "w_wmin") else None,
+ "scale": module.scale if compatible_stored_qtype and hasattr(module, "scale") else None,
+ "zp": module.zp if compatible_stored_qtype and hasattr(module, "zp") else None,
+ "d_scale": module.w_d_scale if compatible_stored_qtype and hasattr(module, "w_d_scale") else None,
+ "d_wmin": module.w_d_wmin if compatible_stored_qtype and hasattr(module, "w_d_wmin") else None,
+ "wmin": module.w_wmin if compatible_stored_qtype and hasattr(module, "w_wmin") else None,
"imatrix": module.imatrix if hasattr(module, "imatrix") else None,
}
if any(isinstance(v, torch.Tensor) for v in kwargs.values()):
@@ -502,6 +515,23 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d
return data, data_qtype
+def _pack_spec_moe_output(cls, data_torch, data_qtype, moe_output, modify_name, new_name, bid, device=None, context=""):
+ def quantize_expert(arr, source_name):
+ return _quant_data(
+ cls,
+ arr,
+ data_qtype,
+ source_name,
+ modify_name,
+ new_name,
+ bid,
+ device=device,
+ use_layer_attrs=True,
+ )
+
+ return pack_moe_output(data_torch, data_qtype, moe_output, quantize_expert, context=context)
+
+
def get_qtype_by_layer_config(layer_config, name, data_qtype, *, explicit_only=False):
name = name[: -len(".weight")]
if name not in layer_config and name.endswith("embed_tokens"):
@@ -715,7 +745,7 @@ def filter_restored_tensor(cls, restored):
filtered = cls.filter_tensors((restored.checkpoint_name, restored.tensor_fn))
if filtered is None:
return None
- return RestoredTensor(filtered[0], filtered[1], restored.hf_names, restored.transform_kind)
+ return RestoredTensor(filtered[0], filtered[1], restored.hf_names, restored.transform_kind, restored.moe_sources)
def _gguf_writer_has_tensor(gguf_writer, tensor_name):
@@ -736,6 +766,8 @@ def _count_attention_wv_tensors(cls):
count = 0
for item in chain(cls.generate_extra_tensors(), cls.get_tensors()):
restored = _to_restored_tensor(item)
+ if restored.moe_sources:
+ continue
filtered = filter_restored_tensor(cls, restored)
if filtered is None:
continue
@@ -786,12 +818,6 @@ def prepare_tensors(cls):
for hf_name in restored.hf_names
):
continue
- data_torch = restored.tensor_fn()
- if data_torch is None or data_torch.numel() == 0:
- continue
- # we don't need these
- if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):
- continue
if hasattr(cls, "current_packing_block") and cls.current_packing_block is not None: # pylint: disable=E1101
current_packing_block_split = cls.current_packing_block.split(".") # pylint: disable=E1101
name_split = name.split(".")
@@ -800,7 +826,12 @@ def prepare_tensors(cls):
or name_split[: len(current_packing_block_split)] != current_packing_block_split
):
continue
-
+ data_torch = restored.tensor_fn()
+ if data_torch is None or data_torch.numel() == 0:
+ continue
+ # we don't need these
+ if name.endswith((".attention.masked_bias", ".attention.bias", ".rotary_emb.inv_freq")):
+ continue
filtered = filter_restored_tensor(cls, restored)
if filtered is None:
continue
@@ -836,6 +867,7 @@ def prepare_tensors(cls):
data_qtype: gguf.GGMLQuantizationType | bool = cls.tensor_force_quant(
checkpoint_name, new_name, bid, n_dims
)
+ moe_output = resolve_moe_output(cls, filtered, new_name, bid)
# Most of the codebase that takes in 1D tensors or norms only handles F32 tensors
if n_dims <= 1 or new_name.endswith("_norm.weight"):
@@ -898,14 +930,29 @@ def prepare_tensors(cls):
data_qtype = gguf.GGMLQuantizationType.F32
layer_config_names = hf_names
else:
- # get name by new_name (for experts),
- layer_config_names = tuple(get_moe_name(cls, source_name, new_name) for source_name in hf_names)
- # get data_qtype by layer_config
fallback_qtype = (
cls._gguf_dtype_selector.select_qtype(new_name, n_dims)
if isinstance(data_qtype, bool)
else data_qtype
)
+ if moe_output is not None:
+ layer_config_names = moe_output.hf_names
+ validate_moe_source_qtypes(
+ layer_config_names,
+ fallback_qtype,
+ lambda source_name: get_qtype_by_layer_config(
+ cls.layer_config, source_name, fallback_qtype, explicit_only=True
+ ),
+ f"{checkpoint_name} -> {new_name}",
+ )
+ validate_moe_imatrices(
+ layer_config_names,
+ lambda source_name: get_module(cls.model, source_name.removesuffix(".weight")),
+ f"{checkpoint_name} -> {new_name}",
+ )
+ else:
+ # Native fused tensors without source metadata retain the legacy name heuristic.
+ layer_config_names = tuple(get_moe_name(cls, source_name, new_name) for source_name in hf_names)
layer_config_qtype = resolve_restored_qtype(
cls.layer_config,
layer_config_names,
@@ -993,7 +1040,19 @@ def prepare_tensors(cls):
pre_quant_shape,
)
- if isinstance(data_qtype, bool) or data_qtype in [
+ if moe_output is not None:
+ data, data_qtype = _pack_spec_moe_output(
+ cls,
+ data_torch,
+ data_qtype,
+ moe_output,
+ modify_name,
+ new_name,
+ bid,
+ device=device,
+ context=f"{checkpoint_name} -> {new_name}",
+ )
+ elif isinstance(data_qtype, bool) or data_qtype in [
gguf.GGMLQuantizationType.F16,
gguf.GGMLQuantizationType.BF16,
gguf.GGMLQuantizationType.F32,
@@ -1039,7 +1098,8 @@ def prepare_tensors(cls):
data = _quant_data_with_args(data_torch, data_qtype, device=device, **attr_list)
# for MOE model
- elif len(data_torch.shape) == 3 and len(re.findall(r"\d+", hf_name)) == 2:
+ # Spec-backed models must not enter this native-fused fallback.
+ elif not restored.moe_sources and len(data_torch.shape) == 3 and len(re.findall(r"\d+", hf_name)) == 2:
# `hf_name` is the checkpoint name of whichever single expert tensor happened
# to complete the buffering in modify_tensors (it collects down_proj/gate_proj/
# up_proj for all experts before yielding); all three merged tensors
diff --git a/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py b/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py
index 8f6b8f14e..6121ef887 100644
--- a/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py
+++ b/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py
@@ -21,6 +21,7 @@ from typing import Callable, Iterator
import torch
+from auto_round.modeling.fused_moe.fusion_spec import MoETensorSource, iter_moe_fusion_views
from auto_round.utils import logger
@@ -30,6 +31,7 @@ class RestoredTensor:
tensor_fn: Callable[[], torch.Tensor]
hf_names: tuple[str, ...]
transform_kind: str
+ moe_sources: tuple[MoETensorSource, ...] = ()
class HFCheckpointRestorer:
@@ -49,12 +51,55 @@ class HFCheckpointRestorer:
def iter_tensors(self) -> Iterator[RestoredTensor]:
state_dict = self.model.state_dict()
+ source_order = {name: index for index, name in enumerate(state_dict)}
+ moe_source_names = set()
+ ordered_tensors = []
+ moe_views = iter_moe_fusion_views(self.model) if hasattr(self.model, "named_modules") else ()
+ for view in moe_views:
+ hf_names = tuple(name for source in view.sources for name in source.hf_names)
+ completed = [name for name in hf_names if name in self.completed_hf_names]
+ if len(completed) == len(hf_names):
+ # Already packed (and freed) by a previous block; the sources are
+ # expected to be gone from state_dict, so skip the presence check.
+ moe_source_names.update(hf_names)
+ continue
+ if completed:
+ incomplete = [name for name in hf_names if name not in self.completed_hf_names]
+ raise ValueError(
+ f"Cannot restore {view.checkpoint_name} from partially completed sources; "
+ f"incomplete sources: {', '.join(incomplete)}"
+ )
+ missing = [name for name in hf_names if name not in source_order]
+ if missing:
+ raise ValueError(
+ f"Cannot restore {view.checkpoint_name}; missing state-dict sources: {', '.join(missing)}"
+ )
+ moe_source_names.update(hf_names)
+ ordered_tensors.append(
+ (
+ min(source_order[name] for name in hf_names),
+ RestoredTensor(
+ checkpoint_name=view.checkpoint_name,
+ tensor_fn=view.tensor_fn,
+ hf_names=hf_names,
+ transform_kind="moe_refusion",
+ moe_sources=view.sources,
+ ),
+ )
+ )
+
weight_conversions = self._get_weight_conversions()
if not weight_conversions:
- for name, tensor in state_dict.items():
+ for index, (name, tensor) in enumerate(state_dict.items()):
+ if name in moe_source_names:
+ continue
if name in self.completed_hf_names:
continue
- yield RestoredTensor(name, lambda tensor=tensor: tensor, (name,), "passthrough")
+ ordered_tensors.append(
+ (index, RestoredTensor(name, lambda tensor=tensor: tensor, (name,), "passthrough"))
+ )
+ for _, restored in sorted(ordered_tensors, key=lambda item: item[0]):
+ yield restored
return
inverted_transforms = self._get_inverted_transforms(weight_conversions)
@@ -70,9 +115,9 @@ class HFCheckpointRestorer:
converter_lineage = defaultdict(list)
converter_fallback_tensors = defaultdict(list)
converter_order = {}
- ordered_tensors = []
-
- for source_order, (original_key, tensor) in enumerate(state_dict.items()):
+ for index, (original_key, tensor) in enumerate(state_dict.items()):
+ if original_key in moe_source_names:
+ continue
converter_key, matched_pattern = self._rename_source_key(original_key, [], inverted_converters)
checkpoint_key, _ = self._rename_source_key(converter_key, inverted_renamings, [])
@@ -82,7 +127,7 @@ class HFCheckpointRestorer:
transform_kind = "rename" if checkpoint_key != original_key else "passthrough"
ordered_tensors.append(
(
- source_order,
+ index,
RestoredTensor(
checkpoint_key,
lambda tensor=tensor: tensor,
@@ -96,7 +141,7 @@ class HFCheckpointRestorer:
mapping = conversion_mapping.setdefault(converter_key, deepcopy(pattern_to_converter[matched_pattern]))
mapping.add_tensor(checkpoint_key, original_key, matched_pattern, tensor)
converter_lineage[converter_key].append(original_key)
- converter_order.setdefault(converter_key, source_order)
+ converter_order.setdefault(converter_key, index)
converter_fallback_tensors[converter_key].append(
RestoredTensor(checkpoint_key, lambda tensor=tensor: tensor, (original_key,), "passthrough")
)
diff --git a/auto_round/export/export_to_gguf/moe_adapter.py b/auto_round/export/export_to_gguf/moe_adapter.py
new file mode 100644
index 000000000..c93843e43
--- /dev/null
+++ b/auto_round/export/export_to_gguf/moe_adapter.py
@@ -0,0 +1,151 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpers for mapping and independently packing restored GGUF MoE outputs."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+import torch
+
+from auto_round.utils import LazyImport
+
+gguf = LazyImport("gguf")
+
+_PROJECTION_ALIASES = {
+ gguf.MODEL_TENSOR.FFN_GATE_EXP: ("gate_proj", "w1", "linear"),
+ gguf.MODEL_TENSOR.FFN_DOWN_EXP: ("down_proj", "w2", "linear_1"),
+ gguf.MODEL_TENSOR.FFN_UP_EXP: ("up_proj", "w3", "linear_v"),
+}
+
+
+@dataclass(frozen=True)
+class GGUFMoEOutput:
+ """The source projection and expert names for one GGUF MoE output."""
+
+ projection: str
+ hf_names: tuple[str, ...]
+
+
+def resolve_moe_output(cls, restored: Any, new_name: str, bid: int | None) -> GGUFMoEOutput | None:
+ """Resolve one GGUF expert output to exactly one restored source projection."""
+
+ sources = restored.moe_sources
+ if not sources:
+ return None
+
+ candidates = []
+ for tensor_type, aliases in _PROJECTION_ALIASES.items():
+ if cls.match_model_tensor_name(new_name, tensor_type, bid):
+ candidates.extend(source for source in sources if source.projection in aliases)
+ if len(candidates) != 1:
+ raise ValueError(f"{new_name}: cannot map GGUF MoE output to checkpoint source projections")
+
+ source = candidates[0]
+ return GGUFMoEOutput(projection=source.projection, hf_names=source.hf_names)
+
+
+def _expert_context(context: str, projection: str, expert_index: int) -> str:
+ return f"{context}, projection {projection}, expert {expert_index}"
+
+
+def pack_moe_output(
+ data_torch: torch.Tensor,
+ data_qtype: gguf.GGMLQuantizationType,
+ output: GGUFMoEOutput,
+ quantize_fn: Callable[[torch.Tensor, str], tuple[np.ndarray, gguf.GGMLQuantizationType]],
+ context: str,
+) -> tuple[np.ndarray, gguf.GGMLQuantizationType]:
+ """Pack each expert independently and stack byte-identical callback results."""
+
+ if data_torch.ndim != 3:
+ error_context = _expert_context(context, output.projection, 0)
+ raise ValueError(f"{error_context}: expected a three-dimensional MoE tensor, got {tuple(data_torch.shape)}")
+ if data_torch.shape[0] != len(output.hf_names):
+ expert_index = min(data_torch.shape[0], len(output.hf_names))
+ error_context = _expert_context(context, output.projection, expert_index)
+ raise ValueError(
+ f"{error_context}: expert count mismatch: tensor has {data_torch.shape[0]}, "
+ f"source lineage has {len(output.hf_names)}"
+ )
+
+ packed = []
+ packed_layout = None
+ for expert_index, (expert_tensor, hf_name) in enumerate(zip(data_torch, output.hf_names)):
+ expert_packed, expert_qtype = quantize_fn(expert_tensor, hf_name)
+ error_context = _expert_context(context, output.projection, expert_index)
+ if not isinstance(expert_packed, np.ndarray):
+ raise ValueError(
+ f"{error_context}: packed result must be a NumPy ndarray, got {type(expert_packed).__name__}"
+ )
+ if expert_qtype != data_qtype:
+ raise ValueError(f"{error_context}: packed qtype mismatch: expected {data_qtype}, got {expert_qtype}")
+ expert_layout = (expert_packed.dtype, expert_packed.shape, expert_packed.nbytes)
+ if packed_layout is None:
+ packed_layout = expert_layout
+ elif expert_layout != packed_layout:
+ raise ValueError(
+ f"{error_context}: packed byte-layout mismatch: expected dtype {packed_layout[0]}, "
+ f"shape {packed_layout[1]}, and {packed_layout[2]} bytes; got dtype {expert_layout[0]}, "
+ f"shape {expert_layout[1]}, and {expert_layout[2]} bytes"
+ )
+ packed.append(expert_packed)
+
+ return np.stack(packed, axis=0), data_qtype
+
+
+def validate_moe_source_qtypes(
+ hf_names: tuple[str, ...],
+ fallback_qtype: gguf.GGMLQuantizationType,
+ resolve_explicit_qtype: Callable[[str], gguf.GGMLQuantizationType | None],
+ context: str,
+) -> gguf.GGMLQuantizationType:
+ """Select one qtype only when all expert sources agree."""
+
+ explicit = [resolve_explicit_qtype(name) for name in hf_names]
+ present = [qtype for qtype in explicit if qtype is not None]
+ if not present:
+ return fallback_qtype
+ if len(present) != len(explicit) or len(set(present)) != 1:
+ raise ValueError(f"{context}: MoE source qtypes are missing or inconsistent: {explicit}")
+ return present[0]
+
+
+def validate_moe_imatrices(
+ hf_names: tuple[str, ...],
+ resolve_module: Callable[[str], Any],
+ context: str,
+) -> None:
+ """Validate the all-absent or all-present per-source imatrix contract."""
+
+ modules = [resolve_module(name) for name in hf_names]
+ imatrices = [getattr(module, "imatrix", None) for module in modules]
+ if not any(imatrix is not None for imatrix in imatrices):
+ return
+
+ for expert_index, (module, imatrix) in enumerate(zip(modules, imatrices)):
+ error_context = f"{context}, expert {expert_index}"
+ if imatrix is None:
+ raise ValueError(f"{error_context}: MoE source imatrix is missing")
+ expected_length = module.weight.shape[-1]
+ if not isinstance(imatrix, torch.Tensor) or imatrix.ndim != 1 or imatrix.shape[0] != expected_length:
+ shape = tuple(imatrix.shape) if isinstance(imatrix, torch.Tensor) else type(imatrix).__name__
+ raise ValueError(
+ f"{error_context}: MoE source imatrix must be one-dimensional with length "
+ f"{expected_length}, got {shape}"
+ )
diff --git a/auto_round/modeling/fused_moe/fusion_spec.py b/auto_round/modeling/fused_moe/fusion_spec.py
new file mode 100644
index 000000000..ec3faf3c9
--- /dev/null
+++ b/auto_round/modeling/fused_moe/fusion_spec.py
@@ -0,0 +1,252 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Format-neutral descriptions and lazy views for reversible MoE fusion."""
+
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import torch
+from torch import nn
+
+_MOE_FUSION_SPEC_ATTRIBUTE = "_auto_round_moe_fusion_spec"
+
+
+@dataclass(frozen=True)
+class ProjectionFusionSpec:
+ """Describe how live expert projections form one checkpoint projection."""
+
+ checkpoint_projection: str
+ source_projections: tuple[str, ...]
+ concat_dim: int | None
+ checkpoint_transposed: bool = False
+ checkpoint_bias: str | None = None
+
+
+@dataclass(frozen=True)
+class MoEFusionSpec:
+ """Describe all reversible projections for one MoE experts module."""
+
+ num_experts: int
+ projections: tuple[ProjectionFusionSpec, ...]
+
+
+@dataclass(frozen=True)
+class MoETensorSource:
+ """Identify the live tensors that contribute to a lazy checkpoint view."""
+
+ projection: str
+ hf_names: tuple[str, ...]
+ parameter: str = "weight"
+
+
+@dataclass(frozen=True)
+class MoEFusionView:
+ """Expose one lazily reconstructed checkpoint tensor and its live sources."""
+
+ checkpoint_name: str
+ tensor_fn: Callable[[], torch.Tensor]
+ sources: tuple[MoETensorSource, ...]
+
+
+def register_moe_fusion_spec(module: nn.Module, spec: MoEFusionSpec) -> None:
+ """Attach a fusion specification as plain module metadata."""
+
+ object.__setattr__(module, _MOE_FUSION_SPEC_ATTRIBUTE, spec)
+
+
+def get_moe_fusion_spec(module: nn.Module) -> MoEFusionSpec | None:
+ """Return fusion metadata previously registered on ``module``."""
+
+ return getattr(module, _MOE_FUSION_SPEC_ATTRIBUTE, None)
+
+
+def build_standard_moe_fusion_spec(
+ detected_projections: Mapping[str, Mapping[str, Any]],
+ num_experts: int,
+ checkpoint_transposed: bool,
+ module: nn.Module,
+) -> MoEFusionSpec:
+ """Build fusion metadata from the standard fused-MoE projection description."""
+
+ projections = []
+ for checkpoint_projection, config in detected_projections.items():
+ source_projections = tuple(config.get("split_into") or (checkpoint_projection,))
+ concat_dim = config.get("concat_dim", 1 if len(source_projections) > 1 else None)
+ checkpoint_bias_name = f"{checkpoint_projection}_bias"
+ checkpoint_bias = checkpoint_bias_name if getattr(module, checkpoint_bias_name, None) is not None else None
+ projections.append(
+ ProjectionFusionSpec(
+ checkpoint_projection=checkpoint_projection,
+ source_projections=source_projections,
+ concat_dim=concat_dim,
+ checkpoint_transposed=checkpoint_transposed,
+ checkpoint_bias=checkpoint_bias,
+ )
+ )
+ return MoEFusionSpec(num_experts=num_experts, projections=tuple(projections))
+
+
+def _qualified_name(module_path: str, name: str) -> str:
+ return f"{module_path}.{name}" if module_path else name
+
+
+def _get_module(module: nn.Module, module_path: str) -> nn.Module | None:
+ try:
+ return module.get_submodule(module_path)
+ except (AttributeError, KeyError):
+ return None
+
+
+def _tensor_sources(
+ module_path: str,
+ projection_spec: ProjectionFusionSpec,
+ num_experts: int,
+ parameter: str = "weight",
+) -> tuple[MoETensorSource, ...]:
+ return tuple(
+ MoETensorSource(
+ projection=projection,
+ hf_names=tuple(
+ _qualified_name(module_path, f"{expert_index}.{projection}.{parameter}")
+ for expert_index in range(num_experts)
+ ),
+ parameter=parameter,
+ )
+ for projection in projection_spec.source_projections
+ )
+
+
+def _context(module_path: str, projection: str, expert_index: int) -> str:
+ return f"module '{module_path or '<root>'}', projection '{projection}', expert {expert_index}"
+
+
+def _stack_source(
+ model: nn.Module,
+ module_path: str,
+ source: MoETensorSource,
+) -> torch.Tensor:
+ tensors = []
+ expected_shape = None
+ for expert_index, hf_name in enumerate(source.hf_names):
+ module_name = hf_name.rsplit(".", 1)[0]
+ source_module = _get_module(model, module_name)
+ tensor = getattr(source_module, source.parameter, None) if source_module is not None else None
+ context = _context(module_path, source.projection, expert_index)
+ if not isinstance(tensor, torch.Tensor):
+ raise ValueError(f"Missing {source.parameter} for {context}")
+ expected_ndim = 2 if source.parameter == "weight" else 1
+ if tensor.ndim != expected_ndim:
+ raise ValueError(
+ f"Incompatible {source.parameter} shape for {context}: "
+ f"expected {expected_ndim} dimensions, got {tuple(tensor.shape)}"
+ )
+ if expected_shape is None:
+ expected_shape = tensor.shape
+ elif tensor.shape != expected_shape:
+ raise ValueError(
+ f"Incompatible {source.parameter} shape for {context}: "
+ f"expected {tuple(expected_shape)}, got {tuple(tensor.shape)}"
+ )
+ tensors.append(tensor)
+ return torch.stack(tensors, dim=0)
+
+
+def _concatenate_sources(
+ tensors: list[torch.Tensor],
+ sources: tuple[MoETensorSource, ...],
+ module_path: str,
+ concat_dim: int | None,
+) -> torch.Tensor:
+ if len(tensors) == 1:
+ return tensors[0]
+ if concat_dim is None:
+ context = _context(module_path, sources[1].projection, 0)
+ raise ValueError(f"Incompatible shape for {context}: multiple sources require concat_dim")
+
+ reference = tensors[0]
+ if not -reference.ndim <= concat_dim < reference.ndim:
+ raise ValueError(
+ f"Incompatible shape for {_context(module_path, sources[0].projection, 0)}: "
+ f"concat dimension {concat_dim} is invalid for {tuple(reference.shape)}"
+ )
+ concat_dim %= reference.ndim
+ for source, tensor in zip(sources[1:], tensors[1:]):
+ if tensor.shape != reference.shape:
+ raise ValueError(
+ f"Incompatible shape for {_context(module_path, source.projection, 0)}: "
+ f"expected {tuple(reference.shape)}, got {tuple(tensor.shape)}"
+ )
+ return torch.cat(tensors, dim=concat_dim)
+
+
+def _build_tensor(
+ model: nn.Module,
+ module_path: str,
+ projection_spec: ProjectionFusionSpec,
+ sources: tuple[MoETensorSource, ...],
+ parameter: str = "weight",
+) -> torch.Tensor:
+ tensors = [_stack_source(model, module_path, source) for source in sources]
+ concat_dim = 1 if parameter == "bias" and len(tensors) > 1 else projection_spec.concat_dim
+ tensor = _concatenate_sources(tensors, sources, module_path, concat_dim)
+ if parameter == "weight" and projection_spec.checkpoint_transposed:
+ if tensor.ndim != 3:
+ raise ValueError(
+ f"Incompatible shape for {_context(module_path, sources[0].projection, 0)}: "
+ f"expected a three-dimensional weight, got {tuple(tensor.shape)}"
+ )
+ tensor = tensor.transpose(1, 2)
+ return tensor
+
+
+def iter_moe_fusion_views(model: nn.Module) -> Iterator[MoEFusionView]:
+ """Yield lazy checkpoint tensor views for every registered MoE module."""
+
+ named_modules = list(model.named_modules())
+ spec_paths = {module_path for module_path, module in named_modules if get_moe_fusion_spec(module) is not None}
+ for module_path, module in named_modules:
+ if not getattr(module, "_auto_round_replaced_fused_moe", False):
+ continue
+ descendant_prefix = f"{module_path}." if module_path else ""
+ has_spec = module_path in spec_paths or any(
+ spec_path.startswith(descendant_prefix) for spec_path in spec_paths if spec_path
+ )
+ if not has_spec:
+ raise ValueError(f"{module_path or '<root>'}: linearized fused MoE replacement has no MoEFusionSpec")
+
+ for module_path, module in named_modules:
+ fusion_spec = get_moe_fusion_spec(module)
+ if fusion_spec is None:
+ continue
+ for projection_spec in fusion_spec.projections:
+ weight_sources = _tensor_sources(module_path, projection_spec, fusion_spec.num_experts)
+ yield MoEFusionView(
+ checkpoint_name=_qualified_name(module_path, projection_spec.checkpoint_projection),
+ tensor_fn=lambda module_path=module_path, projection_spec=projection_spec, sources=weight_sources: (
+ _build_tensor(model, module_path, projection_spec, sources)
+ ),
+ sources=weight_sources,
+ )
+
+ if projection_spec.checkpoint_bias is not None:
+ bias_sources = _tensor_sources(module_path, projection_spec, fusion_spec.num_experts, "bias")
+ yield MoEFusionView(
+ checkpoint_name=_qualified_name(module_path, projection_spec.checkpoint_bias),
+ tensor_fn=lambda module_path=module_path, projection_spec=projection_spec, sources=bias_sources: (
+ _build_tensor(model, module_path, projection_spec, sources, "bias")
+ ),
+ sources=bias_sources,
+ )
diff --git a/auto_round/modeling/fused_moe/gpt_oss.py b/auto_round/modeling/fused_moe/gpt_oss.py
index da68a57fc..370d7ba0d 100644
--- a/auto_round/modeling/fused_moe/gpt_oss.py
+++ b/auto_round/modeling/fused_moe/gpt_oss.py
@@ -26,6 +26,7 @@ else:
from transformers.models.gpt_oss.configuration_gpt_oss import GptOssConfig
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP
+from auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec
from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
from auto_round.modeling.fused_moe.utils import _update_parameter
from auto_round.utils import clear_memory, unsupported_meta_device
@@ -59,6 +60,8 @@ class SequentialGPTOSSMoE(ReplacementModuleBase):
Copies weights from fused tensors and reuses the original router and optional shared_expert.
"""
+ supports_gguf_fused_moe = True
+
def __init__(self, original: GptOssMLP, config: GptOssConfig):
super().__init__(original)
hidden_size = config.hidden_size
@@ -82,6 +85,18 @@ class SequentialGPTOSSMoE(ReplacementModuleBase):
with no_init_weights(), torch.device("meta"):
for _ in range(E):
self.experts.append(GPTOssSingleExpert(hidden_size, intermediate_size, dtype=dtype))
+ register_moe_fusion_spec(
+ self.experts,
+ build_standard_moe_fusion_spec(
+ detected_projections={
+ "gate_up_proj": {"split_into": ["gate_proj", "up_proj"], "concat_dim": 0},
+ "down_proj": {},
+ },
+ num_experts=E,
+ checkpoint_transposed=True,
+ module=original.experts,
+ ),
+ )
def _materialize_weights(self) -> None:
original = self._get_original_module()
diff --git a/auto_round/modeling/fused_moe/llama4.py b/auto_round/modeling/fused_moe/llama4.py
index 342978aac..c6b2f0591 100644
--- a/auto_round/modeling/fused_moe/llama4.py
+++ b/auto_round/modeling/fused_moe/llama4.py
@@ -23,6 +23,7 @@ else:
from transformers.initialization import no_init_weights
from transformers.models.llama4.modeling_llama4 import Llama4Config, Llama4TextMLP
+from auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec
from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
from auto_round.modeling.fused_moe.utils import _update_parameter
from auto_round.utils import clear_memory, unsupported_meta_device
@@ -33,6 +34,18 @@ class SequentialLlama4TextExperts(torch.nn.ModuleList):
self.num_experts = original.gate_up_proj.shape[0]
with no_init_weights(), torch.device("meta"):
super().__init__([Llama4TextMLP(config) for _ in range(self.num_experts)])
+ register_moe_fusion_spec(
+ self,
+ build_standard_moe_fusion_spec(
+ detected_projections={
+ "gate_up_proj": {"split_into": ["gate_proj", "up_proj"]},
+ "down_proj": {},
+ },
+ num_experts=self.num_experts,
+ checkpoint_transposed=True,
+ module=original,
+ ),
+ )
def _materialize_weights(self, original) -> None:
if not unsupported_meta_device(original):
@@ -50,6 +63,8 @@ class SequentialLlama4TextExperts(torch.nn.ModuleList):
class SequentialLlama4TextMoe(ReplacementModuleBase):
+ supports_gguf_fused_moe = True
+
def __init__(self, original, config):
super().__init__(original)
config = config.text_config
diff --git a/auto_round/modeling/fused_moe/moe_experts_interface.py b/auto_round/modeling/fused_moe/moe_experts_interface.py
index f6e901699..61d34da42 100644
--- a/auto_round/modeling/fused_moe/moe_experts_interface.py
+++ b/auto_round/modeling/fused_moe/moe_experts_interface.py
@@ -34,6 +34,7 @@ Usage:
import torch
from torch import nn
+from auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec
from auto_round.utils import clear_memory, logger
from auto_round.utils.device import memory_monitor
@@ -519,6 +520,13 @@ def _unfuse_experts_weights_inplace(
dim1, dim2 = first_param.shape[1], first_param.shape[2]
is_transposed = dim1 < dim2
+ fusion_spec = build_standard_moe_fusion_spec(
+ detected_projections={name: config.copy() for name, config in detected_projections.items()},
+ num_experts=num_experts,
+ checkpoint_transposed=is_transposed,
+ module=module,
+ )
+
dtype = first_param.dtype
target_device = first_param.device if first_param.device.type != "meta" else "cpu"
@@ -591,6 +599,8 @@ def _unfuse_experts_weights_inplace(
if not hasattr(module, "num_experts"):
module.num_experts = num_experts
+ register_moe_fusion_spec(module, fusion_spec)
+
# Mark as unfused for detection during save
module._unfused_experts = True
diff --git a/auto_round/modeling/fused_moe/qwen3_5_moe.py b/auto_round/modeling/fused_moe/qwen3_5_moe.py
index b9fc560a7..1e15a102e 100644
--- a/auto_round/modeling/fused_moe/qwen3_5_moe.py
+++ b/auto_round/modeling/fused_moe/qwen3_5_moe.py
@@ -6,6 +6,7 @@ import torch.nn.functional as F
from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeMLP
from transformers.utils.versions import require_version
+from auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec
from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
from auto_round.utils import clear_memory, unsupported_meta_device
@@ -15,6 +16,8 @@ from auto_round.modeling.fused_moe.utils import _update_parameter
class LinearQwen3_5MoeSparseMoeBlock(ReplacementModuleBase):
+ supports_gguf_fused_moe = True
+
def __init__(self, original, config):
super().__init__(original)
self.gate = original.gate
@@ -95,6 +98,19 @@ class SequentialQwen3_5MoeExperts(torch.nn.ModuleList):
with torch.device("meta"):
super().__init__([Qwen3_5MoeMLP(config, intermediate_size) for _ in range(self.num_experts)])
+ register_moe_fusion_spec(
+ self,
+ build_standard_moe_fusion_spec(
+ detected_projections={
+ "gate_up_proj": {"split_into": ["gate_proj", "up_proj"]},
+ "down_proj": {},
+ },
+ num_experts=self.num_experts,
+ checkpoint_transposed=False,
+ module=original,
+ ),
+ )
+
def _materialize_weights(self, original) -> None:
intermediate_size = original.down_proj.shape[-1]
if not unsupported_meta_device(original):
diff --git a/auto_round/modeling/fused_moe/qwen3_omni.py b/auto_round/modeling/fused_moe/qwen3_omni.py
index 58731f4db..17d3f493d 100644
--- a/auto_round/modeling/fused_moe/qwen3_omni.py
+++ b/auto_round/modeling/fused_moe/qwen3_omni.py
@@ -11,6 +11,7 @@ Qwen3-Omni has MoE blocks in both thinker and talker:
import torch
+from auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec
from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
from auto_round.modeling.fused_moe.utils import _update_parameter
from auto_round.utils import clear_memory, unsupported_meta_device
@@ -29,6 +30,8 @@ class LinearQwen3OmniThinkerSparseMoeBlock(ReplacementModuleBase):
Structure: gate (router) + experts (unfused).
"""
+ supports_gguf_fused_moe = True
+
def __init__(self, original, config):
super().__init__(original)
self.gate = original.gate
@@ -101,6 +104,18 @@ class SequentialQwen3OmniThinkerExperts(torch.nn.ModuleList):
with torch.device("meta"):
super().__init__([Qwen3OmniMoeThinkerTextMLP(config, intermediate_size) for _ in range(self.num_experts)])
+ register_moe_fusion_spec(
+ self,
+ build_standard_moe_fusion_spec(
+ detected_projections={
+ "gate_up_proj": {"split_into": ["gate_proj", "up_proj"], "concat_dim": 0},
+ "down_proj": {},
+ },
+ num_experts=self.num_experts,
+ checkpoint_transposed=False,
+ module=original,
+ ),
+ )
def _materialize_weights(self, original) -> None:
"""Unfuse fused expert weights into individual nn.Linear layers.
diff --git a/auto_round/modeling/fused_moe/qwen3_vl_moe.py b/auto_round/modeling/fused_moe/qwen3_vl_moe.py
index 102b5abd9..87024dfb8 100644
--- a/auto_round/modeling/fused_moe/qwen3_vl_moe.py
+++ b/auto_round/modeling/fused_moe/qwen3_vl_moe.py
@@ -16,6 +16,7 @@ import torch
import transformers
from packaging import version
+from auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec
from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
from auto_round.utils import clear_memory, unsupported_meta_device
@@ -38,6 +39,8 @@ class LinearQwen3VLMoeTextSparseMoeBlock(ReplacementModuleBase):
experts.
"""
+ supports_gguf_fused_moe = True
+
is_permanent = True
def __init__(
@@ -147,6 +150,18 @@ class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):
with torch.device("meta"):
super().__init__([Qwen3VLMoeTextMLP(config, intermediate_size) for _ in range(self.num_experts)])
+ register_moe_fusion_spec(
+ self,
+ build_standard_moe_fusion_spec(
+ detected_projections={
+ "gate_up_proj": {"split_into": ["gate_proj", "up_proj"]},
+ "down_proj": {},
+ },
+ num_experts=self.num_experts,
+ checkpoint_transposed=True,
+ module=original,
+ ),
+ )
def _materialize_weights(self, original) -> None:
intermediate_size = original.down_proj.shape[1]
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index af05f2f13..d6813f9ac 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -44,6 +44,13 @@ BUILTIN_MODULES = {
"qwen3_omni_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_omni"),
}
+_FUSED_MOE_PROJECTION_NAMES = frozenset({"gate_up_proj", "gate_proj", "up_proj", "down_proj", "w1", "w2", "w3"})
+
+
+def _is_fused_moe_parameter(name: str, parameter: torch.Tensor) -> bool:
+ return parameter.ndim == 3 and any(part in _FUSED_MOE_PROJECTION_NAMES for part in name.split("."))
+
+
# Module name prefixes to exclude from MoE unfusing, keyed by model_type.
# Modules under these prefixes stay in their original fused 3D format during
# quantization. ShardWriter expands them to per-expert 2D tensors at save time.
@@ -178,6 +185,7 @@ class ReplacementModuleBase(ABC, torch.nn.Module):
# Registry: module class name -> replacement module class
_replacement_registry: Dict[str, Type["ReplacementModuleBase"]] = {}
+ supports_gguf_fused_moe: bool = False
def __init_subclass__(cls, **kwargs):
"""Automatically register subclasses in the replacement registry."""
@@ -203,6 +211,8 @@ class ReplacementModuleBase(ABC, torch.nn.Module):
def __init__(self, original: torch.nn.Module):
super().__init__()
+ if any(_is_fused_moe_parameter(name, parameter) for name, parameter in original.named_parameters()):
+ self._auto_round_replaced_fused_moe = True
_global_tracker.register_replacement(
name=str(id(self)),
original=original,
@@ -288,6 +298,7 @@ class ReplacementModuleBase(ABC, torch.nn.Module):
def apply_replacements(
model: torch.nn.Module,
auto_detect_moe: bool = True,
+ gguf_export: bool = False,
) -> torch.nn.Module:
"""
Function to apply module replacements to a model.
@@ -301,6 +312,8 @@ def apply_replacements(
model: The model to apply module replacement to (modified in-place).
auto_detect_moe: If True, automatically detect and handle fused MOE modules
(transformers 5.0+ pattern). Default is True.
+ gguf_export: If True, skip custom fused-MoE replacements without an explicit
+ reversible GGUF fusion specification.
Returns:
The model with modules replaced.
@@ -314,7 +327,7 @@ def apply_replacements(
if not _raw_expert_is_logged:
_raw_expert_is_logged = _log_first_moe_block(model, "before replacement")
- _apply_custom_replacements(model)
+ _apply_custom_replacements(model, gguf_export=gguf_export)
if auto_detect_moe and is_transformers_version_greater_or_equal_5():
@@ -330,7 +343,7 @@ def apply_replacements(
@dump_mem_usage("applying custom replacements")
-def _apply_custom_replacements(model: torch.nn.Module) -> list:
+def _apply_custom_replacements(model: torch.nn.Module, gguf_export: bool = False) -> list:
"""Scan model and replace registered modules with custom implementations.
Args:
@@ -352,6 +365,13 @@ def _apply_custom_replacements(model: torch.nn.Module) -> list:
class_name = module.__class__.__name__
if ReplacementModuleBase.is_registered(class_name):
replacement_cls = ReplacementModuleBase.get_replacement_class(class_name)
+ if (
+ gguf_export
+ and any(_is_fused_moe_parameter(name, parameter) for name, parameter in module.named_parameters())
+ and not replacement_cls.supports_gguf_fused_moe
+ ):
+ skipped_modules.append((name, class_name, "skipped because GGUF fusion spec is unavailable"))
+ continue
# If llm-compressor already linearized experts, skip custom replacement.
if hasattr(module, "experts") and is_linearized_layout(module.experts):
skipped_modules.append((name, class_name, "skipped because linearize_moe"))
diff --git a/auto_round/modeling/fused_moe/step3_5_moe.py b/auto_round/modeling/fused_moe/step3_5_moe.py
index 5f622f1e4..f39cd3897 100755
--- a/auto_round/modeling/fused_moe/step3_5_moe.py
+++ b/auto_round/modeling/fused_moe/step3_5_moe.py
@@ -19,6 +19,7 @@ import torch.nn as nn
import torch.nn.functional as F
from transformers.activations import ACT2FN
+from auto_round.modeling.fused_moe.fusion_spec import build_standard_moe_fusion_spec, register_moe_fusion_spec
from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
from auto_round.modeling.fused_moe.utils import _update_parameter
from auto_round.utils import clear_memory, unsupported_meta_device
@@ -56,6 +57,19 @@ class SequentialStep3p5MoeExperts(torch.nn.ModuleList):
with torch.device("meta"):
super().__init__([Step3p5ExpertMLP(hidden_size, intermediate_size, limit) for _ in range(self.num_experts)])
+ register_moe_fusion_spec(
+ self,
+ build_standard_moe_fusion_spec(
+ detected_projections={
+ "gate_proj": {},
+ "up_proj": {},
+ "down_proj": {},
+ },
+ num_experts=self.num_experts,
+ checkpoint_transposed=False,
+ module=original,
+ ),
+ )
def _materialize_weights(self, original) -> None:
"""Split fused MoELinear weights into individual expert nn.Linear weights.
@@ -79,6 +93,8 @@ class LinearStep3p5MoEMLP(ReplacementModuleBase):
"""Replacement for Step3p5MoEMLP that splits fused MoELinear into
individual nn.Linear per expert for quantization support."""
+ supports_gguf_fused_moe = True
+
def __init__(self, original, config=None):
super().__init__(original)
self.num_experts = original.num_experts
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index c73e0e5c4..02bc556ad 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -393,10 +393,8 @@ def _handle_special_model(model):
def update_module(
model, formats: list[OutputFormat] = None, trust_remote_code: bool = True, cleanup_original: bool = True
):
- if formats is not None and any([format_.is_gguf() for format_ in formats]):
- return model
-
- model = apply_replacements(model)
+ gguf_export = formats is not None and any(format_.is_gguf() for format_ in formats)
+ model = apply_replacements(model, gguf_export=gguf_export)
if cleanup_original:
release_original_module_(model)
diff --git a/auto_round/utils/device_manager.py b/auto_round/utils/device_manager.py
index b3aa5be8f..2c8f1dddd 100644
--- a/auto_round/utils/device_manager.py
+++ b/auto_round/utils/device_manager.py
@@ -48,6 +48,7 @@ import contextlib
import functools
import gc
import re
+import sys
from typing import Optional, Union
import torch
@@ -60,6 +61,7 @@ __all__ = [
"device_manager",
"normalize_default_device_map",
"get_ar_device",
+ "default_enable_torch_compile",
"get_current_device_manager",
"get_current_device_type",
"is_device_available",
@@ -892,6 +894,14 @@ def get_ar_device(device_type: Union[None, str, int, torch.device] = None) -> AR
return device_manager.get_ar_device(device_type)
+def default_enable_torch_compile(
+ device: Union[None, str, int, torch.device] = None, platform_name: str | None = None
+) -> bool:
+ """Return the safe torch.compile default for a backend."""
+ device_type = _normalize_device_type(device) or get_current_device_type()
+ return device_type != "xpu" and (platform_name or sys.platform) != "win32"
+
+
def get_current_device_manager() -> ARDevice:
"""Return the :class:`Device` handle for the active backend (or CPU)."""
return device_manager.current()
diff --git a/test/fixtures.py b/test/fixtures.py
index 2f3ba0c40..904b04be8 100644
--- a/test/fixtures.py
+++ b/test/fixtures.py
@@ -322,6 +322,10 @@ def tiny_qwen35_moe_model_path():
config.num_hidden_layers = 1
config.text_config.layer_types = config.text_config.layer_types[: config.text_config.num_hidden_layers]
config.text_config.use_cache = False
+ # This tiny model doesn't materialize the MTP block, so keep block_count aligned
+ # with the actual number of exported layers to avoid a gguf tensor mismatch
+ # (e.g. missing "blk.N.attn_norm.weight") when loading with llama.cpp.
+ config.text_config.mtp_num_hidden_layers = 0
model = Qwen3_5MoeForConditionalGeneration(config)
model.save_pretrained(tiny_model_path)
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name)
diff --git a/test/test_cpu/core/test_init.py b/test/test_cpu/core/test_init.py
index e9f0ccccc..eee0c5a70 100644
--- a/test/test_cpu/core/test_init.py
+++ b/test/test_cpu/core/test_init.py
@@ -5,6 +5,7 @@ from auto_round import AutoRound
from auto_round.auto_scheme import AutoScheme
from auto_round.cli.parser import build_quantize_parser
from auto_round.compressors.base import BaseOrchestrator
+from auto_round.utils.device_manager import default_enable_torch_compile
def test_torch_compile_platform_default_is_deferred():
@@ -37,6 +38,11 @@ def test_auto_scheme_inherits_torch_compile_setting():
assert auto_scheme.enable_torch_compile is None
+def test_xpu_torch_compile_is_disabled_by_default():
+ assert not default_enable_torch_compile("xpu", platform_name="linux")
+ assert default_enable_torch_compile("cuda", platform_name="linux")
+
+
def test_torch_compile_runtime_defaults(tiny_opt_model_path):
ar = AutoRound(model=tiny_opt_model_path, scheme="W4A16", iters=0, nsamples=1)
assert ar.enable_torch_compile
diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.py
index 1cf38e9fd..ae10e8222 100644
--- a/test/test_cpu/export/test_gguf_format.py
+++ b/test/test_cpu/export/test_gguf_format.py
@@ -16,6 +16,30 @@ AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
+def test_update_module_applies_replacements_for_gguf(monkeypatch):
+ from auto_round import special_model_handler
+
+ model = object()
+ replaced_model = object()
+ calls = []
+
+ def apply_replacements(candidate, **kwargs):
+ calls.append((candidate, kwargs))
+ return replaced_model
+
+ class GGUFFormat:
+ @staticmethod
+ def is_gguf():
+ return True
+
+ monkeypatch.setattr(special_model_handler, "apply_replacements", apply_replacements)
+
+ result = special_model_handler.update_module(model, formats=[GGUFFormat()], cleanup_original=False)
+
+ assert result is replaced_model
+ assert calls == [(model, {"gguf_export": True})]
+
+
class TestGGUF:
@classmethod
diff --git a/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py
index 3f76bee91..6f6084b19 100644
--- a/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py
+++ b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py
@@ -4,6 +4,11 @@ from torch import nn
from transformers.core_model_loading import Chunk, PrefixChange, WeightConverter
from auto_round.export.export_to_gguf.hf_checkpoint_restorer import HFCheckpointRestorer
+from auto_round.modeling.fused_moe.fusion_spec import (
+ MoEFusionSpec,
+ ProjectionFusionSpec,
+ register_moe_fusion_spec,
+)
class _DummyConfig:
@@ -21,6 +26,85 @@ class _DummyModel:
return self._state_dict
+class _MoEModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = nn.ModuleList([nn.Module()])
+ self.layers[0].experts = nn.ModuleList(
+ [
+ nn.ModuleDict(
+ {
+ "gate_proj": nn.Linear(2, 2, bias=False),
+ "up_proj": nn.Linear(2, 2, bias=False),
+ "down_proj": nn.Linear(2, 2, bias=False),
+ }
+ )
+ for _ in range(2)
+ ]
+ )
+ register_moe_fusion_spec(
+ self.layers[0].experts,
+ MoEFusionSpec(
+ num_experts=2,
+ projections=(
+ ProjectionFusionSpec(
+ checkpoint_projection="gate_up_proj",
+ source_projections=("gate_proj", "up_proj"),
+ concat_dim=1,
+ ),
+ ProjectionFusionSpec(
+ checkpoint_projection="down_proj",
+ source_projections=("down_proj",),
+ concat_dim=None,
+ ),
+ ),
+ ),
+ )
+
+
+def test_restorer_emits_moe_refusion_view_with_complete_source_lineage():
+ restored = list(HFCheckpointRestorer(_MoEModel()).iter_tensors())
+ by_name = {item.checkpoint_name: item for item in restored}
+ gate_up = by_name["layers.0.experts.gate_up_proj"]
+
+ assert gate_up.transform_kind == "moe_refusion"
+ assert gate_up.hf_names == (
+ "layers.0.experts.0.gate_proj.weight",
+ "layers.0.experts.1.gate_proj.weight",
+ "layers.0.experts.0.up_proj.weight",
+ "layers.0.experts.1.up_proj.weight",
+ )
+ assert [source.projection for source in gate_up.moe_sources] == ["gate_proj", "up_proj"]
+ assert not any(item.checkpoint_name.endswith("0.gate_proj.weight") for item in restored)
+
+
+def test_restorer_rejects_partially_completed_moe_refusion_view():
+ model = _MoEModel()
+ completed = {"layers.0.experts.0.gate_proj.weight"}
+
+ with pytest.raises(ValueError, match="layers.0.experts.gate_up_proj.*incomplete sources"):
+ list(HFCheckpointRestorer(model, completed_hf_names=completed).iter_tensors())
+
+
+def test_get_and_filter_restored_tensor_preserve_moe_sources():
+ from auto_round.export.export_to_gguf.convert import filter_restored_tensor, get_restored_tensors
+
+ class Converter:
+ @staticmethod
+ def filter_tensors(item):
+ return item
+
+ converter = Converter()
+ converter.model = _MoEModel()
+ restored = next(
+ item for item in get_restored_tensors(converter) if item.checkpoint_name == "layers.0.experts.gate_up_proj"
+ )
+ filtered = filter_restored_tensor(converter, restored)
+
+ assert [source.projection for source in restored.moe_sources] == ["gate_proj", "up_proj"]
+ assert filtered.moe_sources == restored.moe_sources
+
+
def test_restorer_reverses_transformers_prefix_change():
model = _DummyModel(
{
diff --git a/test/test_cpu/export/test_gguf_moe_adapter.py b/test/test_cpu/export/test_gguf_moe_adapter.py
new file mode 100644
index 000000000..dfe276666
--- /dev/null
+++ b/test/test_cpu/export/test_gguf_moe_adapter.py
@@ -0,0 +1,305 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import gguf
+import numpy as np
+import pytest
+import torch
+from torch import nn
+
+from auto_round.export.export_to_gguf import convert
+from auto_round.export.export_to_gguf.hf_checkpoint_restorer import RestoredTensor
+from auto_round.export.export_to_gguf.moe_adapter import (
+ GGUFMoEOutput,
+ pack_moe_output,
+ resolve_moe_output,
+ validate_moe_imatrices,
+ validate_moe_source_qtypes,
+)
+from auto_round.modeling.fused_moe.fusion_spec import MoETensorSource
+
+
+class _FakeConverter:
+ _TENSOR_NAMES = {
+ gguf.MODEL_TENSOR.FFN_GATE_EXP: "blk.0.ffn_gate_exps.weight",
+ gguf.MODEL_TENSOR.FFN_UP_EXP: "blk.0.ffn_up_exps.weight",
+ gguf.MODEL_TENSOR.FFN_DOWN_EXP: "blk.0.ffn_down_exps.weight",
+ }
+
+ @classmethod
+ def match_model_tensor_name(cls, name, tensor_type, bid):
+ assert bid == 0
+ return name == cls._TENSOR_NAMES[tensor_type]
+
+
+def _restored(*sources):
+ return RestoredTensor(
+ checkpoint_name="model.layers.0.mlp.experts.gate_up_proj",
+ tensor_fn=lambda: torch.empty(0),
+ hf_names=tuple(name for source in sources for name in source.hf_names),
+ transform_kind="moe_refusion",
+ moe_sources=tuple(sources),
+ )
+
+
+GATE_NAMES = (
+ "model.layers.0.experts.0.gate_proj.weight",
+ "model.layers.0.experts.1.gate_proj.weight",
+)
+UP_NAMES = (
+ "model.layers.0.experts.0.up_proj.weight",
+ "model.layers.0.experts.1.up_proj.weight",
+)
+
+
+ ("new_name", "projection", "hf_names"),
+ [
+ ("blk.0.ffn_gate_exps.weight", "gate_proj", GATE_NAMES),
+ ("blk.0.ffn_up_exps.weight", "up_proj", UP_NAMES),
+ ],
+)
+def test_resolve_moe_output_selects_only_matching_source(new_name, projection, hf_names):
+ restored = _restored(
+ MoETensorSource("gate_proj", GATE_NAMES),
+ MoETensorSource("up_proj", UP_NAMES),
+ )
+
+ output = resolve_moe_output(_FakeConverter, restored, new_name, bid=0)
+
+ assert output == GGUFMoEOutput(projection=projection, hf_names=hf_names)
+
+
+def test_resolve_moe_output_rejects_ambiguous_projection_source():
+ restored = _restored(
+ MoETensorSource("gate_proj", GATE_NAMES),
+ MoETensorSource("w1", UP_NAMES),
+ )
+
+ with pytest.raises(ValueError, match="cannot map GGUF MoE output"):
+ resolve_moe_output(_FakeConverter, restored, "blk.0.ffn_gate_exps.weight", bid=0)
+
+
+class _ExpertModule(nn.Module):
+ def __init__(self, scale, imatrix):
+ super().__init__()
+ self.weight = nn.Parameter(torch.zeros(2, 3))
+ self.scale = scale
+ if imatrix is not None:
+ self.imatrix = imatrix
+
+
+def _expert_modules():
+ return {
+ GATE_NAMES[0]: _ExpertModule(torch.tensor([1.0]), torch.tensor([1.0, 2.0, 3.0])),
+ GATE_NAMES[1]: _ExpertModule(torch.tensor([2.0]), torch.tensor([4.0, 5.0, 6.0])),
+ }
+
+
+def test_pack_moe_output_quantizes_each_expert_with_its_source_metadata():
+ modules = _expert_modules()
+ data = torch.stack([torch.full((2, 3), 1.0), torch.full((2, 3), 2.0)])
+ calls = []
+
+ def quantize_fn(expert_tensor, hf_name):
+ module = modules[hf_name]
+ calls.append((expert_tensor, hf_name, module, module.scale, module.imatrix))
+ return np.full((2, 2), int(expert_tensor[0, 0].item()), dtype=np.uint8), gguf.GGMLQuantizationType.Q4_0
+
+ packed, qtype = pack_moe_output(
+ data,
+ gguf.GGMLQuantizationType.Q4_0,
+ GGUFMoEOutput("gate_proj", GATE_NAMES),
+ quantize_fn,
+ "block 0",
+ )
+
+ reference = np.stack([np.full((2, 2), 1, dtype=np.uint8), np.full((2, 2), 2, dtype=np.uint8)], axis=0)
+ assert np.array_equal(packed, reference)
+ assert qtype == gguf.GGMLQuantizationType.Q4_0
+ assert [call[1] for call in calls] == list(GATE_NAMES)
+ assert calls[0][2] is modules[GATE_NAMES[0]]
+ assert calls[1][2] is modules[GATE_NAMES[1]]
+
+
+def test_pack_moe_output_rejects_expert_count_mismatch():
+ with pytest.raises(ValueError, match="block 0.*projection gate_proj.*expert 1"):
+ pack_moe_output(
+ torch.zeros(1, 2, 3),
+ gguf.GGMLQuantizationType.Q4_0,
+ GGUFMoEOutput("gate_proj", GATE_NAMES),
+ lambda *_: pytest.fail("callback must not run"),
+ "block 0",
+ )
+
+
+def test_pack_spec_moe_output_routes_each_expert_to_its_source(monkeypatch):
+ restored = _restored(MoETensorSource("gate_proj", GATE_NAMES))
+ moe_output = resolve_moe_output(_FakeConverter, restored, "blk.0.ffn_gate_exps.weight", bid=0)
+ data = torch.stack([torch.full((2, 3), 1.0), torch.full((2, 3), 2.0)])
+ calls = []
+
+ def quant_data(cls, arr, data_qtype, name, modify_name, new_name, bid, device=None, use_layer_attrs=True):
+ calls.append((arr, name, use_layer_attrs))
+ return np.full((2, 2), len(calls), dtype=np.uint8), data_qtype
+
+ monkeypatch.setattr(convert, "_quant_data", quant_data)
+
+ packed, qtype = convert._pack_spec_moe_output(
+ _FakeConverter,
+ data,
+ gguf.GGMLQuantizationType.Q4_0,
+ moe_output,
+ "model.layers.0.mlp.experts.gate_up_proj",
+ "blk.0.ffn_gate_exps.weight",
+ 0,
+ device="cpu",
+ context="checkpoint -> blk.0.ffn_gate_exps.weight",
+ )
+
+ assert [call[1] for call in calls] == list(GATE_NAMES)
+ assert all(call[2] is True for call in calls)
+ assert np.array_equal(packed[:, 0, 0], np.array([1, 2], dtype=np.uint8))
+ assert qtype == gguf.GGMLQuantizationType.Q4_0
+
+
+def test_prepare_tensors_cleans_selected_moe_sources_and_marks_full_lineage(monkeypatch):
+ class Expert(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(2, 32))
+ self.scale = torch.ones(2, 1)
+ self.zp = torch.ones(2, 1)
+ self.w_d_scale = torch.ones(2, 1)
+ self.w_d_wmin = torch.ones(2, 1)
+ self.w_wmin = torch.ones(2, 1)
+ self.imatrix = torch.ones(32)
+
+ model = nn.Module()
+ model.model = nn.Module()
+ model.model.layers = nn.ModuleList([nn.Module()])
+ model.model.layers[0].experts = nn.ModuleList([nn.Module(), nn.Module()])
+ for expert in model.model.layers[0].experts:
+ expert.gate_proj = Expert()
+ expert.up_proj = Expert()
+
+ restored = RestoredTensor(
+ checkpoint_name="model.layers.0.mlp.experts.gate_up_proj",
+ tensor_fn=lambda: torch.ones(2, 4, 32),
+ hf_names=GATE_NAMES + UP_NAMES,
+ transform_kind="moe_refusion",
+ moe_sources=(
+ MoETensorSource("gate_proj", GATE_NAMES),
+ MoETensorSource("up_proj", UP_NAMES),
+ ),
+ )
+
+ class Writer:
+ def __init__(self):
+ self.tensors = []
+
+ def add_tensor(self, name, data, raw_dtype):
+ self.tensors.append((name, data, raw_dtype))
+
+ class Converter(_FakeConverter):
+ device = "cpu"
+ _gguf_dtype_selector = object()
+ _gguf_name_resolution_diagnostics = []
+ tensor_map = type("TensorMap", (), {"mapping": {}})()
+ model_arch = gguf.MODEL_ARCH.LLAMA
+ current_packing_block = "model.layers.0"
+ quant_nontext_module = True
+ low_cpu_mem_usage = True
+ ftype = gguf.LlamaFileType.MOSTLY_Q4_0
+
+ def __init__(self):
+ self.model = model
+ self.layer_config = {name.removesuffix(".weight"): {"bits": 4, "sym": True} for name in restored.hf_names}
+ self.gguf_writer = Writer()
+ self._gguf_written_checkpoint_names = set()
+ self._gguf_written_hf_names = set()
+
+ def generate_extra_tensors(self):
+ return ()
+
+ def get_tensors(self):
+ return (restored,)
+
+ def filter_tensors(self, item):
+ return item
+
+ def modify_tensors(self, data, modify_name, bid):
+ yield "blk.0.ffn_gate_exps.weight", data[:, :2]
+ yield "blk.0.ffn_up_exps.weight", data[:, 2:]
+
+ def tensor_force_quant(self, checkpoint_name, new_name, bid, n_dims):
+ return gguf.GGMLQuantizationType.Q4_0
+
+ routed_sources = []
+
+ def pack_spec_moe_output(cls, data, qtype, output, *args, **kwargs):
+ routed_sources.append(output.hf_names)
+ return np.zeros(data.shape, dtype=np.float32), qtype
+
+ monkeypatch.setattr(convert, "_pack_spec_moe_output", pack_spec_moe_output)
+ converter = Converter()
+
+ convert.prepare_tensors(converter)
+
+ assert routed_sources == [GATE_NAMES, UP_NAMES]
+ assert converter._gguf_written_checkpoint_names == {restored.checkpoint_name}
+ assert converter._gguf_written_hf_names == set(restored.hf_names)
+ for source_name in restored.hf_names:
+ module = convert.get_module(model, source_name.removesuffix(".weight"))
+ assert module.weight.numel() == 0
+ assert module.scale is None
+ assert module.imatrix is None
+
+
+def test_validate_moe_source_qtypes_returns_uniform_explicit_qtype():
+ qtype = validate_moe_source_qtypes(
+ GATE_NAMES,
+ gguf.GGMLQuantizationType.Q4_0,
+ lambda _: gguf.GGMLQuantizationType.Q5_0,
+ "block 0",
+ )
+
+ assert qtype == gguf.GGMLQuantizationType.Q5_0
+
+
+ "explicit",
+ [
+ [gguf.GGMLQuantizationType.Q4_0, None],
+ [gguf.GGMLQuantizationType.Q4_0, gguf.GGMLQuantizationType.Q5_0],
+ ],
+)
+def test_validate_moe_source_qtypes_rejects_missing_or_inconsistent_values(explicit):
+ by_name = dict(zip(GATE_NAMES, explicit))
+
+ with pytest.raises(ValueError, match="MoE source qtypes are missing or inconsistent"):
+ validate_moe_source_qtypes(
+ GATE_NAMES,
+ gguf.GGMLQuantizationType.Q4_0,
+ by_name.get,
+ "block 0",
+ )
+
+
+def test_validate_moe_imatrices_rejects_partial_missing():
+ modules = _expert_modules()
+ del modules[GATE_NAMES[1]].imatrix
+
+ with pytest.raises(ValueError, match="block 0.*projection gate_proj.*expert 1"):
+ validate_moe_imatrices(GATE_NAMES, modules.__getitem__, "block 0, projection gate_proj")
diff --git a/test/test_cpu/models/test_moe_experts_interface.py b/test/test_cpu/models/test_moe_experts_interface.py
index 6ac653ebd..6ce9cf29e 100644
--- a/test/test_cpu/models/test_moe_experts_interface.py
+++ b/test/test_cpu/models/test_moe_experts_interface.py
@@ -59,6 +59,7 @@ def test_linear_loop_registration():
def test_unfuse_experts_weights():
"""Test unfusing fused expert weights to nn.Linear layers."""
+ from auto_round.modeling.fused_moe.fusion_spec import get_moe_fusion_spec
from auto_round.modeling.fused_moe.moe_experts_interface import _unfuse_experts_weights_inplace
# Create a mock fused experts module (Mixtral style - not transposed)
@@ -88,6 +89,13 @@ def test_unfuse_experts_weights():
success = _unfuse_experts_weights_inplace(module, check_decorator=False)
assert success, "Failed to unfuse weights"
+ spec = get_moe_fusion_spec(module)
+ assert spec is not None
+ assert spec.num_experts == num_experts
+ assert [item.checkpoint_projection for item in spec.projections] == ["gate_up_proj", "down_proj"]
+ assert spec.projections[0].source_projections == ("gate_proj", "up_proj")
+ assert spec.projections[0].checkpoint_transposed is False
+
# Verify structure - per-expert containers with individual Linear layers
for i in range(num_experts):
expert = getattr(module, str(i))
@@ -113,6 +121,7 @@ def test_unfuse_experts_weights():
def test_unfuse_experts_weights_transposed():
"""Test unfusing transposed expert weights (Llama4/GptOss style)."""
+ from auto_round.modeling.fused_moe.fusion_spec import get_moe_fusion_spec
from auto_round.modeling.fused_moe.moe_experts_interface import _unfuse_experts_weights_inplace
num_experts = 4
@@ -141,6 +150,13 @@ def test_unfuse_experts_weights_transposed():
success = _unfuse_experts_weights_inplace(module, check_decorator=False)
assert success, "Failed to unfuse transposed weights"
+ spec = get_moe_fusion_spec(module)
+ assert spec is not None
+ assert spec.num_experts == num_experts
+ assert [item.checkpoint_projection for item in spec.projections] == ["gate_up_proj", "down_proj"]
+ assert spec.projections[0].source_projections == ("gate_proj", "up_proj")
+ assert spec.projections[0].checkpoint_transposed is True
+
# Verify structure - per-expert containers with individual Linear layers
for i in range(num_experts):
expert = getattr(module, str(i))
diff --git a/test/test_cpu/models/test_moe_fusion_spec.py b/test/test_cpu/models/test_moe_fusion_spec.py
new file mode 100644
index 000000000..1aea96107
--- /dev/null
+++ b/test/test_cpu/models/test_moe_fusion_spec.py
@@ -0,0 +1,237 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import torch
+from torch import nn
+
+from auto_round.modeling.fused_moe.fusion_spec import (
+ MoEFusionSpec,
+ MoETensorSource,
+ ProjectionFusionSpec,
+ build_standard_moe_fusion_spec,
+ get_moe_fusion_spec,
+ iter_moe_fusion_views,
+ register_moe_fusion_spec,
+)
+
+
+class _Expert(nn.Module):
+ def __init__(self, expert_index: int, bias: bool = False):
+ super().__init__()
+ self.gate_proj = nn.Linear(3, 2, bias=bias)
+ self.up_proj = nn.Linear(3, 2, bias=bias)
+ self.down_proj = nn.Linear(2, 3, bias=bias)
+
+ offset = expert_index * 100
+ with torch.no_grad():
+ self.gate_proj.weight.copy_(torch.arange(6).reshape(2, 3) + offset)
+ self.up_proj.weight.copy_(torch.arange(6).reshape(2, 3) + offset + 10)
+ self.down_proj.weight.copy_(torch.arange(6).reshape(3, 2) + offset + 20)
+ if bias:
+ self.gate_proj.bias.copy_(torch.arange(2) + offset + 30)
+ self.up_proj.bias.copy_(torch.arange(2) + offset + 40)
+ self.down_proj.bias.copy_(torch.arange(3) + offset + 50)
+
+
+class _Experts(nn.ModuleList):
+ pass
+
+
+class _Layer(nn.Module):
+ def __init__(self, bias: bool = False):
+ super().__init__()
+ self.experts = _Experts([_Expert(0, bias=bias), _Expert(1, bias=bias)])
+
+
+class _Model(nn.Module):
+ def __init__(self, bias: bool = False):
+ super().__init__()
+ self.layers = nn.ModuleList([_Layer(bias=bias)])
+
+
+def _standard_spec(model: _Model, checkpoint_transposed: bool = False):
+ detected_projections = {
+ "gate_up_proj": {"split_into": ["gate_proj", "up_proj"]},
+ "down_proj": {},
+ }
+ return build_standard_moe_fusion_spec(
+ detected_projections=detected_projections,
+ num_experts=2,
+ checkpoint_transposed=checkpoint_transposed,
+ module=model.layers[0].experts,
+ )
+
+
+def test_non_transposed_views_reconstruct_checkpoint_without_mutating_model():
+ model = _Model()
+ experts = model.layers[0].experts
+ spec = _standard_spec(model)
+ register_moe_fusion_spec(experts, spec)
+
+ gate_up = torch.stack(
+ [torch.cat([experts[i].gate_proj.weight, experts[i].up_proj.weight], dim=0) for i in range(2)], dim=0
+ )
+ down = torch.stack([experts[i].down_proj.weight for i in range(2)], dim=0)
+
+ views = {view.checkpoint_name: view for view in iter_moe_fusion_views(model)}
+
+ assert torch.equal(views["layers.0.experts.gate_up_proj"].tensor_fn(), gate_up)
+ assert torch.equal(views["layers.0.experts.down_proj"].tensor_fn(), down)
+ assert views["layers.0.experts.gate_up_proj"].sources == (
+ MoETensorSource(
+ projection="gate_proj",
+ hf_names=(
+ "layers.0.experts.0.gate_proj.weight",
+ "layers.0.experts.1.gate_proj.weight",
+ ),
+ ),
+ MoETensorSource(
+ projection="up_proj",
+ hf_names=(
+ "layers.0.experts.0.up_proj.weight",
+ "layers.0.experts.1.up_proj.weight",
+ ),
+ ),
+ )
+ assert tuple(experts._modules) == ("0", "1")
+ assert "gate_up_proj" not in experts._parameters
+ assert "_auto_round_moe_fusion_spec" not in experts._modules
+ assert "_auto_round_moe_fusion_spec" not in experts._parameters
+ assert "_auto_round_moe_fusion_spec" not in experts._buffers
+ assert get_moe_fusion_spec(experts) is spec
+
+
+def test_transposed_view_reconstructs_checkpoint_weight():
+ model = _Model()
+ experts = model.layers[0].experts
+ register_moe_fusion_spec(experts, _standard_spec(model, checkpoint_transposed=True))
+ gate_up = torch.stack(
+ [torch.cat([experts[i].gate_proj.weight, experts[i].up_proj.weight], dim=0) for i in range(2)], dim=0
+ )
+
+ views = {view.checkpoint_name: view for view in iter_moe_fusion_views(model)}
+
+ assert torch.equal(views["layers.0.experts.gate_up_proj"].tensor_fn(), gate_up.transpose(1, 2))
+
+
+def test_standard_spec_supports_custom_weight_concat_dimension():
+ model = _Model()
+ spec = build_standard_moe_fusion_spec(
+ detected_projections={"gate_up_proj": {"split_into": ["gate_proj", "up_proj"], "concat_dim": 0}},
+ num_experts=2,
+ checkpoint_transposed=True,
+ module=model.layers[0].experts,
+ )
+
+ assert spec.projections[0].concat_dim == 0
+
+
+def test_bias_views_reconstruct_checkpoint_without_weight_transpose():
+ model = _Model(bias=True)
+ experts = model.layers[0].experts
+ spec = MoEFusionSpec(
+ num_experts=2,
+ projections=(
+ ProjectionFusionSpec(
+ checkpoint_projection="gate_up_proj",
+ source_projections=("gate_proj", "up_proj"),
+ concat_dim=1,
+ checkpoint_transposed=True,
+ checkpoint_bias="gate_up_proj_bias",
+ ),
+ ProjectionFusionSpec(
+ checkpoint_projection="down_proj",
+ source_projections=("down_proj",),
+ concat_dim=None,
+ checkpoint_transposed=True,
+ checkpoint_bias="down_proj_bias",
+ ),
+ ),
+ )
+ register_moe_fusion_spec(experts, spec)
+ gate_up_bias = torch.stack(
+ [torch.cat([experts[i].gate_proj.bias, experts[i].up_proj.bias], dim=0) for i in range(2)], dim=0
+ )
+ down_bias = torch.stack([experts[i].down_proj.bias for i in range(2)], dim=0)
+
+ views = {view.checkpoint_name: view for view in iter_moe_fusion_views(model)}
+
+ assert torch.equal(views["layers.0.experts.gate_up_proj_bias"].tensor_fn(), gate_up_bias)
+ assert torch.equal(views["layers.0.experts.down_proj_bias"].tensor_fn(), down_bias)
+ assert all(source.parameter == "bias" for source in views["layers.0.experts.gate_up_proj_bias"].sources)
+
+
+def test_missing_source_has_contextual_error():
+ model = _Model()
+ experts = model.layers[0].experts
+ register_moe_fusion_spec(experts, _standard_spec(model))
+ del experts[1].up_proj
+
+ views = {view.checkpoint_name: view for view in iter_moe_fusion_views(model)}
+
+ with pytest.raises(ValueError, match=r"layers\.0\.experts.*up_proj.*expert 1"):
+ views["layers.0.experts.gate_up_proj"].tensor_fn()
+
+
+def test_marked_linearized_replacement_without_fusion_spec_is_rejected():
+ model = nn.Module()
+ model.block = nn.Module()
+ model.block._auto_round_replaced_fused_moe = True
+
+ with pytest.raises(ValueError, match=r"^block: linearized fused MoE replacement has no MoEFusionSpec$"):
+ list(iter_moe_fusion_views(model))
+
+
+def test_qwen3_5_replacement_registers_descendant_fusion_spec():
+ from auto_round.modeling.fused_moe.qwen3_5_moe import LinearQwen3_5MoeSparseMoeBlock
+
+ hidden_size = 3
+ intermediate_size = 2
+ num_experts = 2
+
+ class TinyExperts(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_up_proj = nn.Parameter(torch.zeros(num_experts, 2 * intermediate_size, hidden_size))
+ self.down_proj = nn.Parameter(torch.zeros(num_experts, hidden_size, intermediate_size))
+
+ class TinyOriginal(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate = nn.Identity()
+ self.shared_expert = nn.Identity()
+ self.experts = TinyExperts()
+ self.shared_expert_gate = nn.Identity()
+
+ text_config = type(
+ "TextConfig",
+ (),
+ {
+ "hidden_size": hidden_size,
+ "hidden_act": "silu",
+ "moe_intermediate_size": intermediate_size,
+ "num_experts": num_experts,
+ },
+ )()
+ config = type("Config", (), {"get_text_config": lambda self: text_config})()
+
+ replacement = LinearQwen3_5MoeSparseMoeBlock(TinyOriginal(), config)
+
+ assert replacement._auto_round_replaced_fused_moe is True
+ assert get_moe_fusion_spec(replacement.experts) is not None
+ assert [view.checkpoint_name for view in iter_moe_fusion_views(replacement)] == [
+ "experts.gate_up_proj",
+ "experts.down_proj",
+ ]
diff --git a/test/test_cpu/models/test_moe_model.py b/test/test_cpu/models/test_moe_model.py
index d97c28663..91fcd7178 100644
--- a/test/test_cpu/models/test_moe_model.py
+++ b/test/test_cpu/models/test_moe_model.py
@@ -143,8 +143,10 @@ class Step3p5MoEMLP(nn.Module):
self.down_proj = _FakeMoELinear(num_experts, intermediate_size, hidden_size)
-def test_step3p5_moe_replacement():
[email protected]("gguf_export", [False, True])
+def test_step3p5_moe_replacement(gguf_export):
"""Verify that apply_replacements swaps Step3p5MoEMLP → LinearStep3p5MoEMLP."""
+ from auto_round.modeling.fused_moe.fusion_spec import get_moe_fusion_spec, iter_moe_fusion_views
from auto_round.modeling.fused_moe.replace_modules import apply_replacements, materialize_model_
from auto_round.modeling.fused_moe.step3_5_moe import LinearStep3p5MoEMLP, Step3p5ExpertMLP
@@ -158,8 +160,14 @@ def test_step3p5_moe_replacement():
orig_weights = {k: moe.get_parameter(k).clone() for k in ("gate_proj.weight", "up_proj.weight", "down_proj.weight")}
- apply_replacements(model, auto_detect_moe=False)
+ apply_replacements(model, auto_detect_moe=False, gguf_export=gguf_export)
assert isinstance(model.moe, LinearStep3p5MoEMLP)
+ assert get_moe_fusion_spec(model.moe.experts) is not None
+ assert [view.checkpoint_name for view in iter_moe_fusion_views(model.moe)] == [
+ "experts.gate_proj",
+ "experts.up_proj",
+ "experts.down_proj",
+ ]
assert count_modules_by_type(model, Step3p5ExpertMLP) == num_experts
materialize_model_(model)
@@ -171,3 +179,17 @@ def test_step3p5_moe_replacement():
out = model.moe(torch.randn(1, 4, hidden_size))
assert out.shape == (1, 4, hidden_size)
+
+
+def test_step3p5_gguf_skips_replacement_without_fusion_capability(monkeypatch):
+ from auto_round.modeling.fused_moe.replace_modules import apply_replacements
+ from auto_round.modeling.fused_moe.step3_5_moe import LinearStep3p5MoEMLP
+
+ model = nn.Module()
+ model.config = type("Cfg", (), {"model_type": "step3p5"})()
+ model.moe = Step3p5MoEMLP(num_experts=2)
+ monkeypatch.setattr(LinearStep3p5MoEMLP, "supports_gguf_fused_moe", False)
+
+ apply_replacements(model, auto_detect_moe=False, gguf_export=True)
+
+ assert isinstance(model.moe, Step3p5MoEMLP)
diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.py
index 458266b86..15149669a 100644
--- a/test/test_cuda/export/test_gguf_format.py
+++ b/test/test_cuda/export/test_gguf_format.py
@@ -1,3 +1,4 @@
+import importlib.util
import os
import shutil
import sys
@@ -9,9 +10,17 @@ from packaging import version
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
+from auto_round.modeling.fused_moe.fusion_spec import get_moe_fusion_spec
from ...envs import require_gguf
-from ...helpers import eval_generated_prompt, evaluate_accuracy, generate_prompt, get_model_path, save_tiny_model
+from ...helpers import (
+ check_version,
+ eval_generated_prompt,
+ evaluate_accuracy,
+ generate_prompt,
+ get_model_path,
+ save_tiny_model,
+)
AUTO_ROUND_PATH = __file__.split("/")
AUTO_ROUND_PATH = "/".join(AUTO_ROUND_PATH[: AUTO_ROUND_PATH.index("test")])
@@ -37,6 +46,80 @@ class TestAutoRound:
yield
shutil.rmtree(self.save_dir, ignore_errors=True)
+ def _export_qwen35_moe_gguf(self, tiny_qwen35_moe_model_path):
+ import gguf
+
+ from auto_round.modeling.fused_moe.qwen3_5_moe import SequentialQwen3_5MoeExperts
+
+ autoround = AutoRound(
+ tiny_qwen35_moe_model_path,
+ iters=0,
+ nsamples=1,
+ seqlen=8,
+ disable_opt_rtn=True,
+ )
+ quantized_model, save_folder = autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q4_0")
+ expert_count = quantized_model.config.get_text_config().num_experts
+ live_expert_containers = [
+ module for module in quantized_model.modules() if isinstance(module, SequentialQwen3_5MoeExperts)
+ ]
+ assert live_expert_containers, "Qwen3.5 quantization must retain live routed expert containers"
+ expected_projections = {
+ "gate_up_proj": ("gate_proj", "up_proj"),
+ "down_proj": ("down_proj",),
+ }
+ for experts in live_expert_containers:
+ fusion_spec = get_moe_fusion_spec(experts)
+ assert fusion_spec is not None, "Qwen3.5 live experts must carry a registered MoEFusionSpec"
+ assert fusion_spec.num_experts == expert_count
+ assert {
+ projection.checkpoint_projection: projection.source_projections
+ for projection in fusion_spec.projections
+ } == expected_projections
+ assert len(experts) == expert_count
+ assert all(
+ all(hasattr(expert, projection) for projection in ("gate_proj", "up_proj", "down_proj"))
+ for expert in experts
+ )
+
+ text_gguf_files = [
+ os.path.join(save_folder, filename)
+ for filename in os.listdir(save_folder)
+ if filename.endswith(".gguf") and "mmproj" not in filename
+ ]
+ assert len(text_gguf_files) == 1
+ gguf_path = text_gguf_files[0]
+
+ reader = gguf.GGUFReader(gguf_path)
+ for projection in ("gate", "up", "down"):
+ tensors = [tensor for tensor in reader.tensors if tensor.name.endswith(f".ffn_{projection}_exps.weight")]
+ assert tensors, f"missing fused expert {projection} tensor"
+ assert all(len(tensor.shape) == 3 for tensor in tensors)
+ assert all(int(tensor.shape[-1]) == expert_count for tensor in tensors)
+ assert all(tensor.tensor_type == gguf.GGMLQuantizationType.Q4_0 for tensor in tensors)
+ return gguf_path
+
+ @pytest.mark.skipif(not check_version("transformers>=5.2.0"), reason="requires transformers >= 5.2.0")
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+ @require_gguf
+ def test_qwen35_moe_gguf(self, tiny_qwen35_moe_model_path):
+ self._export_qwen35_moe_gguf(tiny_qwen35_moe_model_path)
+
+ @pytest.mark.skip(
+ reason="llama-cpp-python in CI intermittently crashes/fails loading the new 3D fused-MoE GGUF "
+ "tensors for Qwen3.5; skip until the dependency is confirmed/updated to support this format."
+ )
+ @pytest.mark.skipif(not check_version("transformers>=5.2.0"), reason="requires transformers >= 5.2.0")
+ @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+ @pytest.mark.skipif(importlib.util.find_spec("llama_cpp") is None, reason="requires llama_cpp")
+ @require_gguf
+ def test_qwen35_moe_llama_cpp_load(self, tiny_qwen35_moe_model_path):
+ import llama_cpp
+
+ gguf_path = self._export_qwen35_moe_gguf(tiny_qwen35_moe_model_path)
+ model = llama_cpp.Llama(model_path=gguf_path, n_ctx=32, logits_all=False)
+ assert model is not None
+
@require_gguf
def test_gguf_format(self, tiny_qwen_model_path, dataloader):
bits, group_size, sym = 4, 32, False