Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix:reduce memory usage when loading native fused MoE checkpoints (#2382)

Heng Guoa5f6ddd8d059
diff --git a/auto_round/context/model.py b/auto_round/context/model.py
index eeffe50ae..7fb5c2338 100644
--- a/auto_round/context/model.py
+++ b/auto_round/context/model.py
@@ -295,11 +295,12 @@ class ModelContext(BaseContext):
"""Whether to build a meta skeleton instead of a full CPU load.
Deliberately narrow: this only targets ``transformers>=5`` models whose experts are
- stored as a *fused* 3D ``nn.Parameter``. For those, the checkpoint holds one 2D
- tensor per expert, ``from_pretrained`` stacks them into the fused parameter, and
- AutoRound then has to split it back into per-expert ``nn.Linear`` to quantize it --
- and splitting real tensors needs the fused tensor and its per-expert copies alive
- at once, i.e. ~2x one experts module on top of an already fully resident model.
+ stored as a *fused* 3D ``nn.Parameter``. The fused parameter may be produced by a
+ Transformers checkpoint converter from per-expert 2D tensors, or already be stored
+ as 3D ``gate_up_proj``/``down_proj`` tensors in the checkpoint. AutoRound has to
+ split either layout back into per-expert ``nn.Linear`` modules to quantize it, and
+ splitting real tensors needs the fused tensor and its per-expert copies alive at
+ once, i.e. ~2x one experts module on top of an already fully resident model.
Building on meta makes that unfuse allocation-free and lets the tuning loop
materialize one block at a time from the checkpoint. Every other model (dense, or
@@ -323,22 +324,28 @@ class ModelContext(BaseContext):
if config is None:
return False
- from auto_round.modeling.fused_moe.moe_experts_interface import config_has_fused_moe_experts
+ from auto_round.modeling.fused_moe.moe_experts_interface import (
+ _config_model_types,
+ config_has_fused_moe_experts,
+ )
from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES
+ from auto_round.utils.disk_stream_util import checkpoint_has_native_fused_moe_experts
- if not config_has_fused_moe_experts(config):
- return False
# Families with a dedicated replacement drive their own (already memory-aware)
# materialization; do not reroute them through the meta skeleton.
- model_type = getattr(config, "model_type", None)
- if model_type in BUILTIN_MODULES:
- logger.debug("meta-skeleton load skipped: %s has a dedicated MoE replacement", model_type)
+ dedicated_model_type = next(
+ (model_type for model_type in _config_model_types(config) if model_type in BUILTIN_MODULES), None
+ )
+ if dedicated_model_type is not None:
+ logger.debug("meta-skeleton load skipped: %s has a dedicated MoE replacement", dedicated_model_type)
return False
checkpoint_dir = self._resolve_local_checkpoint_dir()
if checkpoint_dir is None:
logger.debug("meta-skeleton load skipped: %s is not addressable as a local directory", self.model)
return False
+ if not config_has_fused_moe_experts(config) and not checkpoint_has_native_fused_moe_experts(checkpoint_dir):
+ return False
self.disk_stream_model_dir = checkpoint_dir
logger.info(
"Fused-MoE checkpoint detected: building a meta skeleton and materializing weights per block "
diff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.py
index 05fefe638..793f59466 100644
--- a/auto_round/utils/disk_stream_util.py
+++ b/auto_round/utils/disk_stream_util.py
@@ -20,7 +20,7 @@ from typing import Dict
import torch
import torch.nn as nn
from accelerate.utils import set_module_tensor_to_device
-from safetensors import safe_open
+from safetensors import SafetensorError, safe_open
logger = logging.getLogger(__name__)
@@ -56,6 +56,12 @@ class SafetensorsIndex:
def read_tensor(self, name: str, device: str = "cpu") -> torch.Tensor:
return self.read_tensors([name], device=device)[name]
+ def tensor_shape(self, name: str) -> tuple[int, ...]:
+ """Return a tensor's shape from its safetensors header without reading its payload."""
+ shard_name = self.weight_map[name]
+ with safe_open(str(self.checkpoint_dir / shard_name), framework="pt") as f:
+ return tuple(f.get_slice(name).get_shape())
+
def read_tensors(self, names: list[str], device: str = "cpu") -> Dict[str, torch.Tensor]:
"""Read several tensors, grouped by shard file so each shard is opened and
closed (unmapped) once regardless of how many tensors are pulled from it."""
@@ -89,6 +95,45 @@ def get_safetensors_index(checkpoint_dir: str) -> "SafetensorsIndex":
return SafetensorsIndex(str(checkpoint_dir))
+def checkpoint_has_native_fused_moe_experts(checkpoint_dir: str) -> bool:
+ """Whether a checkpoint directly stores supported fused 3D expert tensors.
+
+ Unlike :func:`config_has_fused_moe_experts`, which detects experts fused by a
+ Transformers checkpoint converter, this inspects the checkpoint layout itself.
+ Only the ``gate_up_proj`` + ``down_proj`` layout understood by
+ :func:`materialize_module` is accepted. Tensor shapes come from safetensors
+ headers, so this check never maps or copies the weight payloads.
+ """
+ try:
+ index = get_safetensors_index(str(checkpoint_dir))
+ except (OSError, ValueError, KeyError, SafetensorError):
+ return False
+
+ gate_up_suffix = ".experts.gate_up_proj"
+ for gate_up_name in index.weight_map:
+ if not gate_up_name.endswith(gate_up_suffix):
+ continue
+ prefix = gate_up_name[: -len(gate_up_suffix)]
+ down_name = f"{prefix}.experts.down_proj"
+ if not index.has_tensor(down_name):
+ continue
+ try:
+ gate_up_shape = index.tensor_shape(gate_up_name)
+ down_shape = index.tensor_shape(down_name)
+ except (OSError, ValueError, KeyError, SafetensorError):
+ continue
+ if len(gate_up_shape) == 3 and len(down_shape) == 3 and gate_up_shape[0] == down_shape[0] > 1:
+ logger.debug(
+ "Native fused-MoE checkpoint tensors detected: %s %s and %s %s",
+ gate_up_name,
+ gate_up_shape,
+ down_name,
+ down_shape,
+ )
+ return True
+ return False
+
+
# Model-side module names sometimes differ from the checkpoint-side tensor
# names. Transformers already maintains the authoritative per-family renames
# (``transformers/conversion_mapping.py``, checkpoint -> model direction);
diff --git a/test/unit/common/utils/test_disk_stream_util.py b/test/unit/common/utils/test_disk_stream_util.py
index bc20165b3..dd7dafcaf 100644
--- a/test/unit/common/utils/test_disk_stream_util.py
+++ b/test/unit/common/utils/test_disk_stream_util.py
@@ -14,11 +14,15 @@
"""Unit tests for auto_round.utils.disk_stream_util (AR_DISK_STREAM_MODEL primitives)."""
+import json
+
import pytest
import torch
import torch.nn as nn
from accelerate import init_empty_weights
+from safetensors.torch import save_file
+from auto_round.utils import disk_stream_util
from auto_round.utils.disk_stream_util import (
build_meta_model,
free_module,
@@ -28,6 +32,97 @@ from auto_round.utils.disk_stream_util import (
)
+class TestNativeFusedMoeCheckpointDetection:
+ @staticmethod
+ def _write_checkpoint(tmp_path, tensors):
+ checkpoint_dir = tmp_path / "checkpoint"
+ checkpoint_dir.mkdir()
+ save_file(tensors, checkpoint_dir / "model.safetensors")
+ return checkpoint_dir
+
+ def test_detects_paired_three_dimensional_fused_experts(self, tmp_path, monkeypatch):
+ checkpoint_dir = self._write_checkpoint(
+ tmp_path,
+ {
+ "model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4),
+ "model.layers.0.experts.down_proj": torch.empty(4, 4, 4),
+ },
+ )
+
+ real_safe_open = disk_stream_util.safe_open
+
+ class HeaderOnlySafeOpen:
+ def __init__(self, *args, **kwargs):
+ self._context = real_safe_open(*args, **kwargs)
+
+ def __enter__(self):
+ self._file = self._context.__enter__()
+ return self
+
+ def __exit__(self, *args):
+ return self._context.__exit__(*args)
+
+ def keys(self):
+ return self._file.keys()
+
+ def get_slice(self, name):
+ return self._file.get_slice(name)
+
+ def get_tensor(self, name):
+ raise AssertionError(f"native fused detection read tensor payload: {name}")
+
+ monkeypatch.setattr(disk_stream_util, "safe_open", HeaderOnlySafeOpen)
+
+ assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is True
+
+ def test_detects_pair_split_across_shards(self, tmp_path):
+ checkpoint_dir = tmp_path / "checkpoint"
+ checkpoint_dir.mkdir()
+ gate_name = "model.layers.0.experts.gate_up_proj"
+ down_name = "model.layers.0.experts.down_proj"
+ save_file({gate_name: torch.empty(4, 8, 4)}, checkpoint_dir / "model-00001-of-00002.safetensors")
+ save_file({down_name: torch.empty(4, 4, 4)}, checkpoint_dir / "model-00002-of-00002.safetensors")
+ index = {
+ "weight_map": {
+ gate_name: "model-00001-of-00002.safetensors",
+ down_name: "model-00002-of-00002.safetensors",
+ }
+ }
+ (checkpoint_dir / "model.safetensors.index.json").write_text(json.dumps(index))
+
+ assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is True
+
+ def test_malformed_safetensors_is_not_detected(self, tmp_path):
+ checkpoint_dir = tmp_path / "checkpoint"
+ checkpoint_dir.mkdir()
+ (checkpoint_dir / "model.safetensors").write_bytes(b"not a safetensors file")
+
+ assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is False
+
+ @pytest.mark.parametrize(
+ "tensors",
+ [
+ {
+ "model.layers.0.experts.gate_up_proj": torch.empty(8, 4),
+ "model.layers.0.experts.down_proj": torch.empty(4, 4),
+ },
+ {"model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4)},
+ {
+ "model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4),
+ "model.layers.1.experts.down_proj": torch.empty(4, 4, 4),
+ },
+ {
+ "model.layers.0.experts.0.gate_proj.weight": torch.empty(8, 4),
+ "model.layers.0.experts.0.down_proj.weight": torch.empty(4, 4),
+ },
+ ],
+ )
+ def test_rejects_non_native_fused_layouts(self, tmp_path, tensors):
+ checkpoint_dir = self._write_checkpoint(tmp_path, tensors)
+
+ assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is False
+
+
class TestMaterializeModuleRoundTrip:
def test_materialize_then_free_round_trip(self, tiny_opt_model_path):
model, _tokenizer, index = build_meta_model(tiny_opt_model_path)
diff --git a/test/unit/test_cpu/models/test_fused_moe_detection.py b/test/unit/test_cpu/models/test_fused_moe_detection.py
index 1f158c14d..9d82a1948 100644
--- a/test/unit/test_cpu/models/test_fused_moe_detection.py
+++ b/test/unit/test_cpu/models/test_fused_moe_detection.py
@@ -20,9 +20,13 @@ replacement) must keep the ordinary load path, so the blast radius stays small.
"""
import pytest
+import torch
import transformers
from packaging import version
+from safetensors.torch import save_file
+from auto_round import envs
+from auto_round.context.model import ModelContext
from auto_round.modeling.fused_moe.moe_experts_interface import (
_config_model_types,
config_has_fused_moe_experts,
@@ -44,6 +48,26 @@ class _FakeConfig:
setattr(self, name, value)
+def _native_fused_checkpoint(tmp_path):
+ checkpoint_dir = tmp_path / "checkpoint"
+ checkpoint_dir.mkdir()
+ save_file(
+ {
+ "model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4),
+ "model.layers.0.experts.down_proj": torch.empty(4, 4, 4),
+ },
+ checkpoint_dir / "model.safetensors",
+ )
+ return checkpoint_dir
+
+
+def _model_context_for_detection(checkpoint_dir):
+ context = object.__new__(ModelContext)
+ context.model = str(checkpoint_dir)
+ context.disk_stream_model_dir = str(checkpoint_dir)
+ return context
+
+
# Fused 3D experts: `from_pretrained` merges `experts.<i>.<proj>` into one parameter.
FUSED_MOE_TYPES = ["qwen3_moe", "qwen2_moe", "deepseek_v3", "glm4_moe", "phimoe", "mixtral"]
# Dense models have no expert conversion at all.
@@ -98,3 +122,48 @@ def test_families_with_a_dedicated_replacement_are_left_alone():
# These two only get a dedicated replacement on the pre-5.0 linear_loop path.
expected |= {"qwen3_vl_moe", "gpt_oss"}
assert set(BUILTIN_MODULES) == expected
+
+
+def test_native_fused_checkpoint_enables_meta_skeleton_without_merge_converter(tmp_path, monkeypatch):
+ checkpoint_dir = _native_fused_checkpoint(tmp_path)
+ context = _model_context_for_detection(checkpoint_dir)
+ config = _FakeConfig("gemma4", text_config=_FakeConfig("gemma4_text"))
+ monkeypatch.setattr(envs, "AR_DISK_STREAM_MODEL", False)
+ monkeypatch.setattr(envs, "AR_DISABLE_META_LOAD", False)
+ monkeypatch.setattr(envs, "AR_DEBUG_LAYER_NUM", None)
+
+ assert config_has_fused_moe_experts(config) is False, "precondition: Gemma4 has no merge converter"
+ assert context._should_use_meta_skeleton(config) is True
+ assert context.disk_stream_model_dir == str(checkpoint_dir)
+
+
+def test_nested_dedicated_replacement_prevents_native_fused_auto_meta_load(tmp_path, monkeypatch):
+ checkpoint_dir = _native_fused_checkpoint(tmp_path)
+ context = _model_context_for_detection(checkpoint_dir)
+ config = _FakeConfig("wrapper", text_config=_FakeConfig("llama4"))
+ monkeypatch.setattr(envs, "AR_DISK_STREAM_MODEL", False)
+ monkeypatch.setattr(envs, "AR_DISABLE_META_LOAD", False)
+ monkeypatch.setattr(envs, "AR_DEBUG_LAYER_NUM", None)
+
+ assert context._should_use_meta_skeleton(config) is False
+
+
+ "force_disk_stream,disable_meta_load,debug_layer_num,expected",
+ [
+ (True, True, 1, True),
+ (False, True, None, False),
+ (False, False, 1, False),
+ ],
+)
+def test_meta_skeleton_override_precedence(
+ tmp_path, monkeypatch, force_disk_stream, disable_meta_load, debug_layer_num, expected
+):
+ checkpoint_dir = _native_fused_checkpoint(tmp_path)
+ context = _model_context_for_detection(checkpoint_dir)
+ config = _FakeConfig("gemma4", text_config=_FakeConfig("gemma4_text"))
+ monkeypatch.setattr(envs, "AR_DISK_STREAM_MODEL", force_disk_stream)
+ monkeypatch.setattr(envs, "AR_DISABLE_META_LOAD", disable_meta_load)
+ monkeypatch.setattr(envs, "AR_DEBUG_LAYER_NUM", debug_layer_num)
+
+ assert context._should_use_meta_skeleton(config) is expected