PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix:reduce memory usage when loading native fused MoE checkpoints (#2382)
Heng Guoa5f6ddd8d059
diff --git a/auto_round/context/model.py b/auto_round/context/model.pyindex eeffe50ae..7fb5c2338 100644--- a/auto_round/context/model.py+++ b/auto_round/context/model.py@@ -295,11 +295,12 @@ class ModelContext(BaseContext):"""Whether to build a meta skeleton instead of a full CPU load.Deliberately narrow: this only targets ``transformers>=5`` models whose experts are- stored as a *fused* 3D ``nn.Parameter``. For those, the checkpoint holds one 2D- tensor per expert, ``from_pretrained`` stacks them into the fused parameter, and- AutoRound then has to split it back into per-expert ``nn.Linear`` to quantize it --- and splitting real tensors needs the fused tensor and its per-expert copies alive- at once, i.e. ~2x one experts module on top of an already fully resident model.+ stored as a *fused* 3D ``nn.Parameter``. The fused parameter may be produced by a+ Transformers checkpoint converter from per-expert 2D tensors, or already be stored+ as 3D ``gate_up_proj``/``down_proj`` tensors in the checkpoint. AutoRound has to+ split either layout back into per-expert ``nn.Linear`` modules to quantize it, and+ splitting real tensors needs the fused tensor and its per-expert copies alive at+ once, i.e. ~2x one experts module on top of an already fully resident model.Building on meta makes that unfuse allocation-free and lets the tuning loopmaterialize one block at a time from the checkpoint. Every other model (dense, or@@ -323,22 +324,28 @@ class ModelContext(BaseContext):if config is None:return False- from auto_round.modeling.fused_moe.moe_experts_interface import config_has_fused_moe_experts+ from auto_round.modeling.fused_moe.moe_experts_interface import (+ _config_model_types,+ config_has_fused_moe_experts,+ )from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES+ from auto_round.utils.disk_stream_util import checkpoint_has_native_fused_moe_experts- if not config_has_fused_moe_experts(config):- return False# Families with a dedicated replacement drive their own (already memory-aware)# materialization; do not reroute them through the meta skeleton.- model_type = getattr(config, "model_type", None)- if model_type in BUILTIN_MODULES:- logger.debug("meta-skeleton load skipped: %s has a dedicated MoE replacement", model_type)+ dedicated_model_type = next(+ (model_type for model_type in _config_model_types(config) if model_type in BUILTIN_MODULES), None+ )+ if dedicated_model_type is not None:+ logger.debug("meta-skeleton load skipped: %s has a dedicated MoE replacement", dedicated_model_type)return Falsecheckpoint_dir = self._resolve_local_checkpoint_dir()if checkpoint_dir is None:logger.debug("meta-skeleton load skipped: %s is not addressable as a local directory", self.model)return False+ if not config_has_fused_moe_experts(config) and not checkpoint_has_native_fused_moe_experts(checkpoint_dir):+ return Falseself.disk_stream_model_dir = checkpoint_dirlogger.info("Fused-MoE checkpoint detected: building a meta skeleton and materializing weights per block "diff --git a/auto_round/utils/disk_stream_util.py b/auto_round/utils/disk_stream_util.pyindex 05fefe638..793f59466 100644--- a/auto_round/utils/disk_stream_util.py+++ b/auto_round/utils/disk_stream_util.py@@ -20,7 +20,7 @@ from typing import Dictimport torchimport torch.nn as nnfrom accelerate.utils import set_module_tensor_to_device-from safetensors import safe_open+from safetensors import SafetensorError, safe_openlogger = logging.getLogger(__name__)@@ -56,6 +56,12 @@ class SafetensorsIndex:def read_tensor(self, name: str, device: str = "cpu") -> torch.Tensor:return self.read_tensors([name], device=device)[name]+ def tensor_shape(self, name: str) -> tuple[int, ...]:+ """Return a tensor's shape from its safetensors header without reading its payload."""+ shard_name = self.weight_map[name]+ with safe_open(str(self.checkpoint_dir / shard_name), framework="pt") as f:+ return tuple(f.get_slice(name).get_shape())+def read_tensors(self, names: list[str], device: str = "cpu") -> Dict[str, torch.Tensor]:"""Read several tensors, grouped by shard file so each shard is opened andclosed (unmapped) once regardless of how many tensors are pulled from it."""@@ -89,6 +95,45 @@ def get_safetensors_index(checkpoint_dir: str) -> "SafetensorsIndex":return SafetensorsIndex(str(checkpoint_dir))+def checkpoint_has_native_fused_moe_experts(checkpoint_dir: str) -> bool:+ """Whether a checkpoint directly stores supported fused 3D expert tensors.++ Unlike :func:`config_has_fused_moe_experts`, which detects experts fused by a+ Transformers checkpoint converter, this inspects the checkpoint layout itself.+ Only the ``gate_up_proj`` + ``down_proj`` layout understood by+ :func:`materialize_module` is accepted. Tensor shapes come from safetensors+ headers, so this check never maps or copies the weight payloads.+ """+ try:+ index = get_safetensors_index(str(checkpoint_dir))+ except (OSError, ValueError, KeyError, SafetensorError):+ return False++ gate_up_suffix = ".experts.gate_up_proj"+ for gate_up_name in index.weight_map:+ if not gate_up_name.endswith(gate_up_suffix):+ continue+ prefix = gate_up_name[: -len(gate_up_suffix)]+ down_name = f"{prefix}.experts.down_proj"+ if not index.has_tensor(down_name):+ continue+ try:+ gate_up_shape = index.tensor_shape(gate_up_name)+ down_shape = index.tensor_shape(down_name)+ except (OSError, ValueError, KeyError, SafetensorError):+ continue+ if len(gate_up_shape) == 3 and len(down_shape) == 3 and gate_up_shape[0] == down_shape[0] > 1:+ logger.debug(+ "Native fused-MoE checkpoint tensors detected: %s %s and %s %s",+ gate_up_name,+ gate_up_shape,+ down_name,+ down_shape,+ )+ return True+ return False++# Model-side module names sometimes differ from the checkpoint-side tensor# names. Transformers already maintains the authoritative per-family renames# (``transformers/conversion_mapping.py``, checkpoint -> model direction);diff --git a/test/unit/common/utils/test_disk_stream_util.py b/test/unit/common/utils/test_disk_stream_util.pyindex bc20165b3..dd7dafcaf 100644--- a/test/unit/common/utils/test_disk_stream_util.py+++ b/test/unit/common/utils/test_disk_stream_util.py@@ -14,11 +14,15 @@"""Unit tests for auto_round.utils.disk_stream_util (AR_DISK_STREAM_MODEL primitives)."""+import json+import pytestimport torchimport torch.nn as nnfrom accelerate import init_empty_weights+from safetensors.torch import save_file+from auto_round.utils import disk_stream_utilfrom auto_round.utils.disk_stream_util import (build_meta_model,free_module,@@ -28,6 +32,97 @@ from auto_round.utils.disk_stream_util import ()+class TestNativeFusedMoeCheckpointDetection:+ @staticmethod+ def _write_checkpoint(tmp_path, tensors):+ checkpoint_dir = tmp_path / "checkpoint"+ checkpoint_dir.mkdir()+ save_file(tensors, checkpoint_dir / "model.safetensors")+ return checkpoint_dir++ def test_detects_paired_three_dimensional_fused_experts(self, tmp_path, monkeypatch):+ checkpoint_dir = self._write_checkpoint(+ tmp_path,+ {+ "model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4),+ "model.layers.0.experts.down_proj": torch.empty(4, 4, 4),+ },+ )++ real_safe_open = disk_stream_util.safe_open++ class HeaderOnlySafeOpen:+ def __init__(self, *args, **kwargs):+ self._context = real_safe_open(*args, **kwargs)++ def __enter__(self):+ self._file = self._context.__enter__()+ return self++ def __exit__(self, *args):+ return self._context.__exit__(*args)++ def keys(self):+ return self._file.keys()++ def get_slice(self, name):+ return self._file.get_slice(name)++ def get_tensor(self, name):+ raise AssertionError(f"native fused detection read tensor payload: {name}")++ monkeypatch.setattr(disk_stream_util, "safe_open", HeaderOnlySafeOpen)++ assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is True++ def test_detects_pair_split_across_shards(self, tmp_path):+ checkpoint_dir = tmp_path / "checkpoint"+ checkpoint_dir.mkdir()+ gate_name = "model.layers.0.experts.gate_up_proj"+ down_name = "model.layers.0.experts.down_proj"+ save_file({gate_name: torch.empty(4, 8, 4)}, checkpoint_dir / "model-00001-of-00002.safetensors")+ save_file({down_name: torch.empty(4, 4, 4)}, checkpoint_dir / "model-00002-of-00002.safetensors")+ index = {+ "weight_map": {+ gate_name: "model-00001-of-00002.safetensors",+ down_name: "model-00002-of-00002.safetensors",+ }+ }+ (checkpoint_dir / "model.safetensors.index.json").write_text(json.dumps(index))++ assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is True++ def test_malformed_safetensors_is_not_detected(self, tmp_path):+ checkpoint_dir = tmp_path / "checkpoint"+ checkpoint_dir.mkdir()+ (checkpoint_dir / "model.safetensors").write_bytes(b"not a safetensors file")++ assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is False++ @pytest.mark.parametrize(+ "tensors",+ [+ {+ "model.layers.0.experts.gate_up_proj": torch.empty(8, 4),+ "model.layers.0.experts.down_proj": torch.empty(4, 4),+ },+ {"model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4)},+ {+ "model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4),+ "model.layers.1.experts.down_proj": torch.empty(4, 4, 4),+ },+ {+ "model.layers.0.experts.0.gate_proj.weight": torch.empty(8, 4),+ "model.layers.0.experts.0.down_proj.weight": torch.empty(4, 4),+ },+ ],+ )+ def test_rejects_non_native_fused_layouts(self, tmp_path, tensors):+ checkpoint_dir = self._write_checkpoint(tmp_path, tensors)++ assert disk_stream_util.checkpoint_has_native_fused_moe_experts(checkpoint_dir) is False++class TestMaterializeModuleRoundTrip:def test_materialize_then_free_round_trip(self, tiny_opt_model_path):model, _tokenizer, index = build_meta_model(tiny_opt_model_path)diff --git a/test/unit/test_cpu/models/test_fused_moe_detection.py b/test/unit/test_cpu/models/test_fused_moe_detection.pyindex 1f158c14d..9d82a1948 100644--- a/test/unit/test_cpu/models/test_fused_moe_detection.py+++ b/test/unit/test_cpu/models/test_fused_moe_detection.py@@ -20,9 +20,13 @@ replacement) must keep the ordinary load path, so the blast radius stays small."""import pytest+import torchimport transformersfrom packaging import version+from safetensors.torch import save_file+from auto_round import envs+from auto_round.context.model import ModelContextfrom auto_round.modeling.fused_moe.moe_experts_interface import (_config_model_types,config_has_fused_moe_experts,@@ -44,6 +48,26 @@ class _FakeConfig:setattr(self, name, value)+def _native_fused_checkpoint(tmp_path):+ checkpoint_dir = tmp_path / "checkpoint"+ checkpoint_dir.mkdir()+ save_file(+ {+ "model.layers.0.experts.gate_up_proj": torch.empty(4, 8, 4),+ "model.layers.0.experts.down_proj": torch.empty(4, 4, 4),+ },+ checkpoint_dir / "model.safetensors",+ )+ return checkpoint_dir+++def _model_context_for_detection(checkpoint_dir):+ context = object.__new__(ModelContext)+ context.model = str(checkpoint_dir)+ context.disk_stream_model_dir = str(checkpoint_dir)+ return context++# Fused 3D experts: `from_pretrained` merges `experts.<i>.<proj>` into one parameter.FUSED_MOE_TYPES = ["qwen3_moe", "qwen2_moe", "deepseek_v3", "glm4_moe", "phimoe", "mixtral"]# Dense models have no expert conversion at all.@@ -98,3 +122,48 @@ def test_families_with_a_dedicated_replacement_are_left_alone():# These two only get a dedicated replacement on the pre-5.0 linear_loop path.expected |= {"qwen3_vl_moe", "gpt_oss"}assert set(BUILTIN_MODULES) == expected+++def test_native_fused_checkpoint_enables_meta_skeleton_without_merge_converter(tmp_path, monkeypatch):+ checkpoint_dir = _native_fused_checkpoint(tmp_path)+ context = _model_context_for_detection(checkpoint_dir)+ config = _FakeConfig("gemma4", text_config=_FakeConfig("gemma4_text"))+ monkeypatch.setattr(envs, "AR_DISK_STREAM_MODEL", False)+ monkeypatch.setattr(envs, "AR_DISABLE_META_LOAD", False)+ monkeypatch.setattr(envs, "AR_DEBUG_LAYER_NUM", None)++ assert config_has_fused_moe_experts(config) is False, "precondition: Gemma4 has no merge converter"+ assert context._should_use_meta_skeleton(config) is True+ assert context.disk_stream_model_dir == str(checkpoint_dir)+++def test_nested_dedicated_replacement_prevents_native_fused_auto_meta_load(tmp_path, monkeypatch):+ checkpoint_dir = _native_fused_checkpoint(tmp_path)+ context = _model_context_for_detection(checkpoint_dir)+ config = _FakeConfig("wrapper", text_config=_FakeConfig("llama4"))+ monkeypatch.setattr(envs, "AR_DISK_STREAM_MODEL", False)+ monkeypatch.setattr(envs, "AR_DISABLE_META_LOAD", False)+ monkeypatch.setattr(envs, "AR_DEBUG_LAYER_NUM", None)++ assert context._should_use_meta_skeleton(config) is False+++ "force_disk_stream,disable_meta_load,debug_layer_num,expected",+ [+ (True, True, 1, True),+ (False, True, None, False),+ (False, False, 1, False),+ ],+)+def test_meta_skeleton_override_precedence(+ tmp_path, monkeypatch, force_disk_stream, disable_meta_load, debug_layer_num, expected+):+ checkpoint_dir = _native_fused_checkpoint(tmp_path)+ context = _model_context_for_detection(checkpoint_dir)+ config = _FakeConfig("gemma4", text_config=_FakeConfig("gemma4_text"))+ monkeypatch.setattr(envs, "AR_DISK_STREAM_MODEL", force_disk_stream)+ monkeypatch.setattr(envs, "AR_DISABLE_META_LOAD", disable_meta_load)+ monkeypatch.setattr(envs, "AR_DEBUG_LAYER_NUM", debug_layer_num)++ assert context._should_use_meta_skeleton(config) is expected