PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
# Copyright (c) 2025 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
import inspect
import json
import os
import re
import sys
from collections import UserDict
from collections.abc import Callable
from pathlib import Path
from typing import TYPE_CHECKING, Any
import psutil
import torch
import transformers
from packaging import version
from auto_round import envs
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.logger import logger
from auto_round.utils.common import AUDIO_MM_KEYS, VISION_MM_KEYS, monkey_patch_model
from auto_round.utils.path_safety import UnsafeCheckpointPathError
from auto_round.utils.weight_handler import (
_dequant_fp8_linear_weight,
check_and_mark_quantized_module,
convert_module_to_hp_if_necessary,
is_quantized_input_module,
)
# Maps architecture class names to virtual model_type keys.
# Used when config.model_type doesn't uniquely identify the model (e.g. MiMo-Audio
# has model_type="qwen2" but needs audio-specific handling).
ARCHITECTURE_MODEL_TYPE_MAP = {
"MiMoAudioModel": "mimo_audio",
"MiMoAudioForCausalLM": "mimo_audio",
}
def resolve_model_type(model):
"""Resolve the effective model type using architecture class name as primary source.
This function prioritizes the model's architecture class name (from config.architectures)
over config.model_type to handle models where the two diverge (e.g., MiMo-Audio has
architecture="MiMoAudioModel" but model_type="qwen2" on HuggingFace).
Args:
model: A model instance with optional config attribute.
Returns:
str or None: The resolved model type identifier, or None if config is missing.
"""
config = getattr(model, "config", None)
if config is None:
return None
# Check architecture-based override first
archs = getattr(config, "architectures", None)
if archs:
for arch in archs:
if arch in ARCHITECTURE_MODEL_TYPE_MAP:
return ARCHITECTURE_MODEL_TYPE_MAP[arch]
return getattr(config, "model_type", None)
FIX_MISTRAL_REGEX_MODEL_TYPE_LIST = ["longcat_next"]
if TYPE_CHECKING:
from auto_round.schemes import QuantizationScheme
def clean_module_parameter(submodule: torch.nn.Module, param_name: str) -> None:
"""This function is recommended to be used instead of module.weight = None.
For models like `tie_word_embeddings`, setting the embedding weight to None
causes `lm_head` to reallocate memory for its weight instead of treating it as a "bound shared weight,"
it's now iterated over as an independent parameter,
resulting in an additional `lm_head` parameter in `named_parameters`.
Args:
submodule (torch.nn.Module): submodule to clean
param_name (str): "weight" or "bias"
"""
if submodule is None:
return
is_buffer = param_name in submodule._buffers
with torch.no_grad():
if is_buffer:
buf = submodule._buffers[param_name]
if buf is not None:
buf.data = torch.empty(0, dtype=buf.dtype, device=buf.device)
buf.requires_grad = False
else:
param = submodule._parameters[param_name]
if param is not None:
param.data = torch.empty(0, dtype=param.dtype, device=param.device)
param.requires_grad = False
def convert_dtype_str2torch(str_dtype):
"""Converts a string dtype to its corresponding PyTorch dtype.
Args:
str_dtype (str): The string representation of the dtype.
Returns:
torch.dtype: The PyTorch dtype.
Raises:
ValueError: If the input str_dtype is unsupported.
"""
if isinstance(str_dtype, torch.dtype) or str_dtype is None:
return str_dtype
if str_dtype == "int8":
return torch.int8
elif str_dtype == "fp32" or str_dtype == "float32" or str_dtype == "auto":
return torch.float
elif str_dtype == "fp16" or str_dtype == "float16":
return torch.float16
elif str_dtype == "bf16" or str_dtype == "bfloat16":
return torch.bfloat16
else:
raise ValueError(f"Unsupported string dtype '{str_dtype}' for conversion to torch dtype.")
def convert_dtype_torch2str(dtype):
"""Converts a PyTorch dtype to its corresponding string representation.
Args:
dtype: PyTorch dtype or str. The dtype to convert.
Returns:
str: The string representation of the dtype.
Raises:
ValueError: If the input dtype is unsupported.
"""
if isinstance(dtype, str) or dtype is None:
return dtype
if dtype == torch.int8:
return "int8"
elif dtype == torch.float:
return "fp32"
elif dtype == torch.float16:
return "fp16"
elif dtype == torch.bfloat16:
return "bf16"
elif isinstance(dtype, str) and dtype in ["int8", "fp32", "fp16", "bf16"]:
return dtype
else:
raise ValueError(f"Unsupported PyTorch dtype '{dtype}' for conversion to string dtype.")
def convert_dtype_torch2str_hf(dtype):
"""Converts a PyTorch dtype to its corresponding huggingface string dtype, e.g. torch.float32 -> 'float32'.
Args:
dtype: PyTorch dtype or str. The dtype to convert.
Returns:
str: The string representation of the dtype.
Raises:
ValueError: If the input str_dtype is unsupported.
"""
if dtype is None:
return dtype
if isinstance(dtype, str):
if "float" not in dtype and "int" not in dtype:
dtype = convert_dtype_str2torch(dtype)
else:
return dtype
str_dtype = str(dtype)
if "." not in str_dtype:
raise ValueError(f"Unsupported pytorch dtype '{dtype}' for conversion to huggingface str dtype")
str_dtype = str_dtype.split(".")[1]
return str_dtype
def check_diffusers_installed(): # pragma: no cover
try:
import diffusers # noqa: F401
return True
except ImportError:
logger.error("Please install diffusers via 'pip install diffusers'" " to run diffusion model")
sys.exit(-1)
def check_start_with_block_name(name: str, block_name_to_quantize: list):
"""
Checks if the given layer name starts with any of the block names to be quantized.
Args:
name (str): The name of the layer.
block_name_to_quantize (list): A list of block names to check against.
Returns:
bool: True if the layer name starts with any of the block names, False otherwise.
"""
for block_name in block_name_to_quantize:
if name.startswith(block_name):
return True
return False
def download_or_get_path(repo_id: str, platform: str | None = None, revision: str | None = None) -> str:
from auto_round import envs
if platform is None:
if envs.AR_USE_MODELSCOPE:
platform = "model_scope"
else:
platform = "hf"
if platform == "model_scope":
return download_modelscope_model(repo_id, revision=revision)
else:
return download_hf_model(repo_id, revision=revision)
def download_modelscope_model(
repo_id: str, local_dir: str | None = None, cache_dir: str | None = None, revision: str | None = None
):
from modelscope.utils.file_utils import get_modelscope_cache_dir # pylint: disable=E0401
system_cache = cache_dir if cache_dir is not None else get_modelscope_cache_dir()
if local_dir:
directory = os.path.abspath(local_dir)
elif cache_dir:
directory = os.path.join(system_cache, *repo_id.split("/"))
else:
directory = os.path.join(system_cache, "models", *repo_id.split("/"))
if os.path.exists(directory) and revision is None:
return directory
else:
from modelscope.hub.snapshot_download import snapshot_download # pylint: disable=E0401
return snapshot_download(repo_id, revision=revision)
def download_hf_model(repo_id, cache_dir=None, repo_type=None, revision=None):
"""Download hugging face model from hf hub."""
from huggingface_hub.constants import DEFAULT_REVISION, HUGGINGFACE_HUB_CACHE
from huggingface_hub.file_download import REGEX_COMMIT_HASH, repo_folder_name
if cache_dir is None:
cache_dir = HUGGINGFACE_HUB_CACHE
if revision is None:
revision = DEFAULT_REVISION
if repo_type is None:
repo_type = "model"
storage_folder = os.path.join(cache_dir, repo_folder_name(repo_id=repo_id, repo_type=repo_type))
commit_hash = None
if REGEX_COMMIT_HASH.match(revision):
commit_hash = revision
else:
ref_path = os.path.join(storage_folder, "refs", revision)
if os.path.exists(ref_path):
with open(ref_path) as f:
commit_hash = f.read()
if storage_folder and commit_hash:
pointer_path = os.path.join(storage_folder, "snapshots", commit_hash)
if os.path.isdir(pointer_path):
return pointer_path
else: # pragma: no cover
from huggingface_hub import snapshot_download
model_path = snapshot_download(repo_id, revision=revision)
return model_path
def _check_accelerate_version():
from auto_round.utils.common import get_library_version
accelerate_version = get_library_version("accelerate")
from packaging.version import Version
if Version(accelerate_version) > Version("1.5.1") and Version(accelerate_version) < Version("1.10.0"):
logger.warning(
f"Detected accelerate version {accelerate_version}. "
"Versions between 1.5.1 and 1.10.0 may cause high RAM usage during model loading. "
"It is recommended to upgrade to version 1.10.0 or above."
)
_MXFP4_SUPPORTED_MODEL_TYPES = {"gpt_oss"}
_FP8_SUPPORTED_MODEL_TYPES = {"deepseek_v32"}
def _is_mxfp4_model(model_path, trust_remote_code=True):
"""Check if a model is an MXFP4 quantized model supported for direct loading.
Only checks when transformers >= 5.0.0. Returns False immediately for older versions,
adding zero overhead to non-MXFP4 model loading.
"""
from transformers import AutoConfig
try: # in case of config loading failure for new models
config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)
except Exception:
return False
model_type = getattr(config, "model_type", "")
if model_type not in _MXFP4_SUPPORTED_MODEL_TYPES:
return False
quant_config = getattr(config, "quantization_config", None)
if quant_config is None:
return False
quant_method = (
quant_config.get("quant_method", "")
if isinstance(quant_config, dict)
else getattr(quant_config, "quant_method", "")
)
return quant_method == "mxfp4" and model_type in _MXFP4_SUPPORTED_MODEL_TYPES
def _is_fp8_model(model_path, trust_remote_code=True):
"""Check if a model is an FP8 quantized model supported for direct loading.
Only checks when transformers >= 4.56.0. Returns False immediately for older versions,
adding zero overhead to non-FP8 model loading.
"""
if version.parse(transformers.__version__) < version.parse("4.56.0"):
return False
from transformers import AutoConfig
try: # in case of config loading failure for new models
config = AutoConfig.from_pretrained(model_path, trust_remote_code=trust_remote_code)
except Exception:
return False
model_type = getattr(config, "model_type", "")
if model_type not in _FP8_SUPPORTED_MODEL_TYPES:
return False
quant_config = getattr(config, "quantization_config", None)
if quant_config is None:
return False
quant_method = (
quant_config.get("quant_method", "")
if isinstance(quant_config, dict)
else getattr(quant_config, "quant_method", "")
)
return quant_method == "fp8" and model_type in _FP8_SUPPORTED_MODEL_TYPES
def _maybe_truncate_debug_layers(config) -> bool:
"""Debug helper: truncate the number of decoder layers on a config in place.
Controlled by the ``AR_DEBUG_LAYER_NUM`` env var (exposed on the CLI as
``--num_hidden_layers``). When set to a positive integer N, only the first N
decoder layers are kept so that very large models can be loaded quickly with
a fraction of the memory for isolating and debugging issues.
Handles both flat configs (``num_hidden_layers``) and the nested sub-configs
used by multimodal models (e.g. ``text_config`` on a Qwen*-VL config). Every
reachable ``PretrainedConfig`` that exposes ``num_hidden_layers`` is truncated.
Returns ``True`` if the config was modified.
"""
debug_layer_num = envs.AR_DEBUG_LAYER_NUM
if debug_layer_num is None or config is None:
return False
# Collect the config itself plus any nested sub-config it carries. Multimodal
# models keep the decoder layer count on a sub-config (text_config /
# thinker_config / ...), so scan the attribute dict generically instead of
# hard-coding one name.
targets = [config]
seen = {id(config)}
for cfg in targets:
for value in list(getattr(cfg, "__dict__", {}).values()):
if id(value) not in seen and (
isinstance(value, transformers.PretrainedConfig) or hasattr(value, "num_hidden_layers")
):
targets.append(value)
seen.add(id(value))
changed = False
for cfg in targets:
current = getattr(cfg, "num_hidden_layers", None)
if isinstance(current, int) and debug_layer_num < current:
cfg.num_hidden_layers = debug_layer_num
changed = True
if changed:
logger.warning_once(
f"AR_DEBUG_LAYER_NUM={debug_layer_num} is set: loading only the first {debug_layer_num} "
"decoder layer(s) for debugging. The resulting model is partial and must not be used for a "
"real quantization run."
)
return changed
def install_debug_layer_config_patch() -> None:
"""Make ``AR_DEBUG_LAYER_NUM`` apply to *every* model-loading path.
The various loaders (``llm_load_model`` / ``mllm_load_model`` /
``diffusion_load_model`` / the meta-skeleton builder) each let ``transformers``
resolve the config internally, so there is no single AutoRound call site to
intercept. They do all funnel through ``PretrainedConfig.from_dict`` (used by
``AutoConfig.from_pretrained``, every concrete config's ``from_pretrained``,
and therefore every ``model.from_pretrained``), so patch that one classmethod
to truncate the decoder-layer count on the freshly built config.
No-op unless ``AR_DEBUG_LAYER_NUM`` is set. Idempotent.
"""
if envs.AR_DEBUG_LAYER_NUM is None:
return
from transformers import PretrainedConfig
original = PretrainedConfig.__dict__.get("from_dict")
if original is None or getattr(original.__func__, "_ar_debug_patched", False):
return
original_func = original.__func__
def _patched_from_dict(cls, config_dict, **kwargs):
result = original_func(cls, config_dict, **kwargs)
# from_dict returns either the config or a (config, unused_kwargs) tuple.
config = result[0] if isinstance(result, tuple) else result
try:
_maybe_truncate_debug_layers(config)
except Exception as exc: # pragma: no cover - best-effort debug path
logger.debug(f"AR_DEBUG_LAYER_NUM truncation skipped for {cls}: {exc}")
return result
_patched_from_dict._ar_debug_patched = True
PretrainedConfig.from_dict = classmethod(_patched_from_dict)
logger.debug("Installed AR_DEBUG_LAYER_NUM config patch on PretrainedConfig.from_dict")
def llm_load_model(
pretrained_model_name_or_path: str,
platform: str = "hf",
trust_remote_code: bool = True,
model_dtype: str | None = None,
device: str = "cpu",
**kwargs,
):
assert platform.lower() in [
"hf",
"model_scope",
], "current only support hf or model_scope platform to load pretrained model."
if platform.lower() == "model_scope" and not envs.AR_USE_MODELSCOPE:
envs.set_config(AR_USE_MODELSCOPE=True)
_check_accelerate_version()
if platform == "model_scope":
from modelscope import AutoModel, AutoModelForCausalLM, AutoTokenizer # pylint: disable=E0401
else:
from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer
from auto_round.utils.device import (
_use_hpu_compile_mode,
fake_cuda_for_hpu,
fake_triton_for_hpu,
is_hpex_available,
override_cuda_device_capability,
)
from auto_round.utils.device_manager import get_device_and_parallelism
device_str, use_auto_mapping = get_device_and_parallelism(device)
torch_dtype = "auto"
if device_str is not None and "hpu" in device_str:
torch_dtype = torch.bfloat16
if "dtype" in kwargs:
torch_dtype = kwargs.pop("dtype")
kwargs.pop("torch_dtype", None)
elif "torch_dtype" in kwargs:
torch_dtype = kwargs.pop("torch_dtype")
dtype_key = (
"dtype"
if platform == "hf" and version.parse(transformers.__version__) >= version.parse("4.56.0")
else "torch_dtype"
)
load_kwargs = {
dtype_key: torch_dtype,
"trust_remote_code": trust_remote_code,
"device_map": "auto" if use_auto_mapping else None,
}
load_kwargs.update(kwargs)
# Debug helper: honor AR_DEBUG_LAYER_NUM (load only the first N decoder
# layers) on every load path via a transformers config patch.
install_debug_layer_config_patch()
if version.parse(transformers.__version__) >= version.parse("4.56.0"):
is_fp8 = _is_fp8_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
if is_fp8 and "quantization_config" not in load_kwargs:
from transformers import FineGrainedFP8Config
load_kwargs["quantization_config"] = FineGrainedFP8Config(dequantize=True)
logger.info("Detected FP8 quantized model, using FineGrainedFP8Config(dequantize=True) for loading.")
if version.parse(transformers.__version__) >= version.parse("5.0.0"):
is_mxfp4 = _is_mxfp4_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
if is_mxfp4:
from transformers import Mxfp4Config
load_kwargs["quantization_config"] = Mxfp4Config(dequantized=True)
logger.info("Detected MXFP4 quantized model, using Mxfp4Config(dequantized=True) for loading.")
# BAGEL requires a custom loader (Qwen2 + not extensions, not in transformers)
_config_path = (
os.path.join(pretrained_model_name_or_path, "config.json")
if os.path.isdir(pretrained_model_name_or_path)
else None
)
if _config_path and os.path.exists(_config_path):
with open(_config_path) as _f:
_mt = json.load(_f).get("model_type")
if _mt == "bagel":
from auto_round.utils.bagel_loader import load_bagel_model
model, tokenizer = load_bagel_model(
pretrained_model_name_or_path,
torch_dtype=torch_dtype,
)
model = _to_model_dtype(model, model_dtype)
model._autoround_to_quant_block_names = "language_model.model.layers"
return model, tokenizer
is_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))
tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
model_cls = AutoModel if is_glm else AutoModelForCausalLM
if "deepseek" in pretrained_model_name_or_path.lower() and trust_remote_code:
logger.warning("trust_remote_code is enabled by default, please ensure its correctness.")
if is_hpex_available():
# For loading FP8 model on HPU
with fake_cuda_for_hpu(), fake_triton_for_hpu(), override_cuda_device_capability():
model = model_cls.from_pretrained(pretrained_model_name_or_path, **load_kwargs)
else:
try:
model = model_cls.from_pretrained(pretrained_model_name_or_path, **load_kwargs)
except ValueError as e:
if "FP8 quantized" in str(e):
with override_cuda_device_capability():
model = model_cls.from_pretrained(pretrained_model_name_or_path, **load_kwargs)
logger.warning("the support for fp8 model as input is experimental, please use with caution.")
else:
raise
except OSError as e:
logger.warning(f"fail to load {pretrained_model_name_or_path}, set trust_remote_code to False and retry.")
model = model_cls.from_pretrained(
pretrained_model_name_or_path, **{**load_kwargs, "trust_remote_code": False}
)
model = model.eval()
monkey_patch_model(model)
check_and_mark_quantized_module(model)
handle_generation_config(model)
_keep_checkpoint_fp32_params(model, pretrained_model_name_or_path, platform, revision=load_kwargs.get("revision"))
model = _to_model_dtype(model, model_dtype)
return model, tokenizer
def _keep_checkpoint_fp32_params(model, pretrained_model_name_or_path, platform, subfolder=None, revision=None):
"""Keep FP32 tensors of a mixed-precision checkpoint in FP32 after ``from_pretrained``."""
from auto_round.utils.missing_tensors import keep_special_fp32_params
revision = getattr(getattr(model, "config", None), "_commit_hash", None) or revision
try:
source_dir = pretrained_model_name_or_path
if not os.path.isdir(source_dir):
source_dir = download_or_get_path(source_dir, platform=platform, revision=revision)
if subfolder is not None:
source_dir = os.path.join(source_dir, subfolder)
keep_special_fp32_params(model, source_dir)
except Exception as e:
logger.warning(f"Skipping restore of FP32 tensors from the checkpoint due to error: {e}")
def _find_pipeline_model_subfolder(model_dir_or_repo: str, file_list: list | None = None) -> tuple:
"""Find model/processor subfolders from a pipeline's model_index.json.
Works for both local directories and remote HF repos.
Args:
model_dir_or_repo: Local directory path or HF repo id.
file_list: If provided, treat *model_dir_or_repo* as a remote HF repo
and use *file_list* (from ``list_repo_files``) to check file existence.
If ``None``, treat it as a local directory.
Returns:
(model_subfolder, processor_subfolder, config_dict)
"""
is_local = file_list is None
if is_local:
index_path = os.path.join(model_dir_or_repo, "model_index.json")
if not os.path.exists(index_path):
raise FileNotFoundError(f"No config.json or model_index.json found under {model_dir_or_repo}")
else:
from huggingface_hub import hf_hub_download
index_path = hf_hub_download(model_dir_or_repo, "model_index.json")
with open(index_path, "r", encoding="utf-8") as f:
model_index = json.load(f)
processor_subfolder = None
for name, value in model_index.items():
if name == "processor" and isinstance(value, list):
processor_subfolder = "processor"
break
candidates = []
for name, value in model_index.items():
if name.startswith("_") or not isinstance(value, list) or len(value) < 2:
continue
# The chosen name becomes a read subfolder and later an output subfolder.
if not name.isidentifier():
raise UnsafeCheckpointPathError(f"model_index.json: component name {name!r} is not a plain directory name")
# Load component config.json
if is_local:
cfg_path = os.path.join(model_dir_or_repo, name, "config.json")
if not os.path.isfile(cfg_path):
continue
with open(cfg_path, "r", encoding="utf-8") as f:
comp_config = json.load(f)
else:
comp_config_file = f"{name}/config.json"
if comp_config_file not in file_list:
continue
cfg_path = hf_hub_download(model_dir_or_repo, comp_config_file)
with open(cfg_path, "r", encoding="utf-8") as f:
comp_config = json.load(f)
if "architectures" in comp_config:
candidates.append((name, comp_config))
if not candidates:
raise FileNotFoundError(
f"model_index.json found in {model_dir_or_repo} but no component with 'architectures' in its config.json"
)
for name, comp_config in candidates:
arch = comp_config["architectures"][0]
if "CausalLM" in arch or "ConditionalGeneration" in arch:
return name, processor_subfolder, comp_config
return candidates[0][0], processor_subfolder, candidates[0][1]
def mllm_load_model(
pretrained_model_name_or_path: str,
platform: str = "hf",
device: str = "cpu",
torch_dtype: str = "auto",
use_auto_mapping: bool = True,
trust_remote_code: bool = True,
model_dtype: str | None = None,
**kwargs,
):
from auto_round.modeling.nemotron_h_omni import (
nemotron_h_omni_load_kwargs,
prefer_remote_omni_class,
register_nemotron_h_omni_checkpoint_mapping,
)
from auto_round.special_model_handler import MISTRAL_3_2_MODELS
_check_accelerate_version()
assert platform.lower() in [
"hf",
"model_scope",
], "current only support hf or model_scope platform to load pretrained model."
if platform.lower() == "model_scope" and not envs.AR_USE_MODELSCOPE:
envs.set_config(AR_USE_MODELSCOPE=True)
if platform == "model_scope":
import modelscope # pylint: disable=E0401
from modelscope import ( # pylint: disable=E0401
AutoConfig,
AutoModel,
AutoModelForCausalLM,
AutoProcessor,
AutoTokenizer,
)
base_lib = modelscope
else:
import transformers
from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoProcessor, AutoTokenizer
base_lib = transformers
from auto_round.utils.device import override_cuda_device_capability
from auto_round.utils.device_manager import get_device_and_parallelism
device_str, use_auto_mapping = get_device_and_parallelism(device)
torch_dtype = "auto"
if device_str is not None and "hpu" in device_str:
torch_dtype = torch.bfloat16
model_subfolder = None
processor_subfolder = None
if os.path.isdir(pretrained_model_name_or_path):
config_path = os.path.join(pretrained_model_name_or_path, "config.json")
if os.path.exists(config_path):
with open(config_path, "r", encoding="utf-8") as f:
config = json.load(f)
else:
model_subfolder, processor_subfolder, config = _find_pipeline_model_subfolder(pretrained_model_name_or_path)
else:
from huggingface_hub import hf_hub_download, list_repo_files
file_list = list_repo_files(pretrained_model_name_or_path)
if "config.json" in file_list:
config_path = hf_hub_download(pretrained_model_name_or_path, "config.json")
with open(config_path, "r", encoding="utf-8") as f:
config = json.load(f)
elif "model_index.json" in file_list:
model_subfolder, processor_subfolder, config = _find_pipeline_model_subfolder(
pretrained_model_name_or_path, file_list
)
elif "config.json.gz" in file_list:
# Load gzipped JSON
import gzip
config_path = hf_hub_download(pretrained_model_name_or_path, "config.json.gz")
with gzip.open(config_path, "rt", encoding="utf-8") as f:
config = json.load(f)
else:
raise FileNotFoundError(f"No config.json or config.json.gz found for {pretrained_model_name_or_path}")
if "model_type" in config:
model_type = config["model_type"]
else:
model_type = None
register_nemotron_h_omni_checkpoint_mapping(config, trust_remote_code)
if model_type == "qwen2_5_omni":
if version.parse(transformers.__version__) < version.parse("4.52.0"):
raise RuntimeError(
f"Qwen2.5-Omni requires transformers >= 4.52.0, but found {transformers.__version__}. "
"Please upgrade: pip install transformers>=4.52.0"
)
if model_type == "qwen3_omni_moe":
if version.parse(transformers.__version__) < version.parse("5.1.0"):
raise RuntimeError(
f"Qwen3-Omni requires transformers >= 5.1.0, but found {transformers.__version__}. "
"Please upgrade: pip install transformers>=5.1.0"
)
processor, image_processor = None, None
if "qwen3_tts" == model_type:
try:
from qwen_tts.core.models import Qwen3TTSConfig, Qwen3TTSForConditionalGeneration
from qwen_tts.core.models.processing_qwen3_tts import Qwen3TTSProcessor
AutoConfig.register("qwen3_tts", Qwen3TTSConfig)
AutoModel.register(Qwen3TTSConfig, Qwen3TTSForConditionalGeneration)
AutoModelForCausalLM.register(Qwen3TTSConfig, Qwen3TTSForConditionalGeneration)
AutoProcessor.register(Qwen3TTSConfig, Qwen3TTSProcessor)
except ImportError:
raise ImportError("Qwen3-TTS requires the 'qwen-tts' package. " "Please install it: pip install qwen-tts")
except TypeError as e:
if "check_model_inputs" in str(e):
raise ImportError(
f"Qwen3-TTS 'qwen-tts' package is incompatible with transformers {transformers.__version__}. "
"Please upgrade qwen-tts: pip install -U qwen-tts"
) from e
raise
# MiMo-Audio: architectures=["MiMoAudioModel"] but model_type="qwen2".
# Requires MiMo-Audio SDK from https://github.com/XiaomiMiMo/MiMo-Audio
# Set MIMO_AUDIO_PATH env var to the cloned repo root (containing src/mimo_audio/).
architectures = config.get("architectures", [])
_is_mimo_audio = any(a in ("MiMoAudioModel", "MiMoAudioForCausalLM") for a in architectures)
if _is_mimo_audio:
try:
from mimo_audio.modeling_mimo_audio import MiMoAudioArguments, MiMoAudioForCausalLM
except ImportError:
# Try adding MIMO_AUDIO_PATH/src to sys.path
mimo_path = os.environ.get("MIMO_AUDIO_PATH")
if mimo_path:
import sys
src_path = os.path.join(mimo_path, "src")
if src_path not in sys.path:
sys.path.insert(0, src_path)
try:
from mimo_audio.modeling_mimo_audio import MiMoAudioArguments, MiMoAudioForCausalLM
except ImportError:
raise ImportError(
"MiMo-Audio requires the MiMo-Audio SDK. "
"Please clone it: git clone https://github.com/XiaomiMiMo/MiMo-Audio.git "
"and set MIMO_AUDIO_PATH to the repo root."
)
else:
raise ImportError(
"MiMo-Audio requires the MiMo-Audio SDK. "
"Please clone https://github.com/XiaomiMiMo/MiMo-Audio and set env var "
"MIMO_AUDIO_PATH to the repo root (e.g. export MIMO_AUDIO_PATH=/path/to/MiMo-Audio)."
)
tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
# Ensure special tokens are registered
special_tokens = ["<|sosp|>", "<|eosp|>", "<|empty|>", "<|sostm|>", "<|eostm|>", "<|eot|>"]
for token in special_tokens:
if token not in tokenizer.get_vocab():
tokenizer.add_tokens([token], special_tokens=True)
model_args = MiMoAudioArguments(
model_name_or_path=pretrained_model_name_or_path,
sosp_idx=tokenizer.convert_tokens_to_ids("<|sosp|>"),
eosp_idx=tokenizer.convert_tokens_to_ids("<|eosp|>"),
sostm_idx=tokenizer.convert_tokens_to_ids("<|sostm|>"),
eostm_idx=tokenizer.convert_tokens_to_ids("<|eostm|>"),
eot_idx=tokenizer.convert_tokens_to_ids("<|eot|>"),
empty_idx=tokenizer.convert_tokens_to_ids("<|empty|>"),
)
model = MiMoAudioForCausalLM.from_pretrained(
pretrained_model_name_or_path,
args=model_args,
torch_dtype=torch_dtype,
device_map="auto" if use_auto_mapping else None,
)
processor = None
elif "deepseek_vl_v2" == model_type:
from deepseek_vl2.models import DeepseekVLV2ForCausalLM, DeepseekVLV2Processor # pylint: disable=E0401
processor = DeepseekVLV2Processor.from_pretrained(pretrained_model_name_or_path)
tokenizer = processor.tokenizer
model: DeepseekVLV2ForCausalLM = AutoModelForCausalLM.from_pretrained(
pretrained_model_name_or_path,
trust_remote_code=trust_remote_code,
torch_dtype=torch_dtype,
device_map="auto" if use_auto_mapping else None,
)
elif "bagel" == model_type:
from auto_round.utils.bagel_loader import load_bagel_model
model, tokenizer = load_bagel_model(
pretrained_model_name_or_path,
torch_dtype=torch_dtype,
)
processor = None
image_processor = None
else:
architectures = config["architectures"][0]
if architectures == "LlavaLlamaForCausalLM":
from llava.model.builder import load_pretrained_model # pylint: disable=E0401
tokenizer, model, image_processor, _ = load_pretrained_model(
pretrained_model_name_or_path,
model_base=None,
model_name=pretrained_model_name_or_path,
torch_dtype=torch_dtype,
)
else:
if prefer_remote_omni_class(config, trust_remote_code):
cls = AutoModelForCausalLM
elif architectures.endswith("Model") and hasattr(
base_lib, n := architectures.replace("Model", "ForConditionalGeneration")
):
cls = getattr(base_lib, n)
elif hasattr(base_lib, architectures):
cls = getattr(base_lib, architectures)
else:
cls = AutoModelForCausalLM
try:
model_load_kwargs = nemotron_h_omni_load_kwargs(config)
if model_subfolder is not None:
model_load_kwargs["subfolder"] = model_subfolder
model = cls.from_pretrained(
pretrained_model_name_or_path,
trust_remote_code=trust_remote_code,
torch_dtype=torch_dtype,
device_map="auto" if use_auto_mapping else None,
**model_load_kwargs,
)
except ValueError as e:
if "FP8 quantized" in str(e):
with override_cuda_device_capability():
model_load_kwargs = {}
if model_subfolder is not None:
model_load_kwargs["subfolder"] = model_subfolder
model = cls.from_pretrained(
pretrained_model_name_or_path,
trust_remote_code=trust_remote_code,
torch_dtype=torch_dtype,
device_map="auto" if use_auto_mapping else None,
**model_load_kwargs,
)
logger.warning("the support for fp8 model as input is experimental, please use with caution.")
else:
raise
if any(name in model.name_or_path for name in MISTRAL_3_2_MODELS):
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # pylint: disable=E0401
if os.path.isdir(pretrained_model_name_or_path):
tokenizer = MistralTokenizer.from_file(os.path.join(pretrained_model_name_or_path, "tekken.json"))
else:
tokenizer = MistralTokenizer.from_hf_hub(pretrained_model_name_or_path)
else:
processor_load_kwargs = {}
if processor_subfolder is not None:
processor_load_kwargs["subfolder"] = processor_subfolder
tokenizer = AutoTokenizer.from_pretrained(
pretrained_model_name_or_path,
trust_remote_code=trust_remote_code,
fix_mistral_regex=model_type in FIX_MISTRAL_REGEX_MODEL_TYPE_LIST,
**processor_load_kwargs,
)
processor = AutoProcessor.from_pretrained(
pretrained_model_name_or_path,
trust_remote_code=trust_remote_code,
**processor_load_kwargs,
)
try:
if platform == "model_scope":
from modelscope import AutoImageProcessor # pylint: disable=E0401
else:
from transformers import AutoImageProcessor
image_processor_load_kwargs = {}
if processor_subfolder is not None:
image_processor_load_kwargs["subfolder"] = processor_subfolder
image_processor = AutoImageProcessor.from_pretrained(
pretrained_model_name_or_path,
trust_remote_code=trust_remote_code,
**image_processor_load_kwargs,
)
except Exception as e:
pass
if model_type == "glm_image" and image_processor is not None:
from transformers.models.glm_image.processing_glm_image import GlmImageProcessor
processor = GlmImageProcessor(image_processor=image_processor, tokenizer=tokenizer)
model = model.eval()
check_and_mark_quantized_module(model)
handle_generation_config(model)
_keep_checkpoint_fp32_params(
model, pretrained_model_name_or_path, platform, model_subfolder, revision=kwargs.get("revision")
)
model = _to_model_dtype(model, model_dtype)
if model_subfolder is not None:
model._autoround_pipeline_subfolder = model_subfolder
return model, processor, tokenizer, image_processor
def _attach_diffusion_pipeline_fn(pipe):
"""Attach a custom pipeline function for diffusion models that need special API calls."""
pipe_class_name = type(pipe).__name__
if pipe_class_name == "StableAudioPipeline":
def _stable_audio_pipeline_fn(
pipe, prompts, guidance_scale=7.0, num_inference_steps=100, generator=None, **kwargs
):
audio_end_in_s = kwargs.pop("audio_end_in_s", 10.0)
return pipe(
prompts,
guidance_scale=guidance_scale,
num_inference_steps=num_inference_steps,
audio_end_in_s=audio_end_in_s,
generator=generator,
)
pipe._autoround_pipeline_fn = _stable_audio_pipeline_fn
def diffusion_load_model(
pretrained_model_name_or_path: str,
platform: str = "hf",
device: str | torch.device = "cpu",
torch_dtype: str | torch.dtype = "auto",
use_auto_mapping: bool = False,
trust_remote_code: bool = True,
model_dtype: str | None = None,
default_torch_dtype: str | torch.dtype = "auto",
**kwargs,
):
from functools import partial
from auto_round.utils.common import LazyImport
from auto_round.utils.device_manager import get_device_and_parallelism
_check_accelerate_version()
if platform != "hf":
raise NotImplementedError(
f"auto_round current only support hf as platform for diffusion model, but get {platform}"
)
device_str, use_auto_mapping = get_device_and_parallelism(device)
if model_dtype is not None:
torch_dtype = convert_dtype_str2torch(model_dtype)
elif torch_dtype == "auto" and device_str is not None and "hpu" in device_str:
torch_dtype = torch.bfloat16
try:
from transformers import AutoConfig
config = AutoConfig.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
except Exception:
config = None
model_type = getattr(config, "model_type", "")
# A special case for NextStep
if model_type == "nextstep":
from auto_round.special_model_handler import load_next_step_diffusion
pipe, model = load_next_step_diffusion(pretrained_model_name_or_path, device_str)
return pipe, pipe.model
# A special case for Cosmos3: model_index.json _class_name may not match
# diffusers, and the safety checker requires cosmos_guardrail which may not be installed.
_model_index = None
if isinstance(pretrained_model_name_or_path, str):
_mi_path = os.path.join(pretrained_model_name_or_path, "model_index.json")
if os.path.isfile(_mi_path):
with open(_mi_path, "r", encoding="utf-8") as f:
_model_index = json.load(f)
_pipe_class = (_model_index or {}).get("_class_name", "")
if _pipe_class in ("Cosmos3OmniDiffusersPipeline", "Cosmos3OmniPipeline"):
from auto_round.special_model_handler import load_cosmos3_diffusion
return load_cosmos3_diffusion(pretrained_model_name_or_path, device_str)
pipelines = LazyImport("diffusers.pipelines")
modular_pipeline_cls = _get_modular_pipeline_class()
if isinstance(pretrained_model_name_or_path, str):
model_index = os.path.join(pretrained_model_name_or_path, "model_index.json")
if not os.path.exists(model_index) and os.path.exists(
os.path.join(pretrained_model_name_or_path, MODULAR_PIPELINE_INDEX_NAME)
):
raise NotImplementedError(
f"{pretrained_model_name_or_path} is a Modular Diffusers pipeline "
f"({MODULAR_PIPELINE_INDEX_NAME}), which auto_round cannot assemble from a path yet. "
"Build the ModularPipeline yourself and pass the pipeline object as `model` instead."
)
with open(model_index, "r", encoding="utf-8") as file:
config = json.load(file)
if torch_dtype == "auto":
torch_dtype = {}
for k, v in config.items():
component_folder = os.path.join(pretrained_model_name_or_path, k)
if isinstance(v, list) and os.path.exists(os.path.join(component_folder, "config.json")):
with open(os.path.join(component_folder, "config.json"), "r", encoding="utf-8") as file:
component_config = json.load(file)
component_dtype = component_config.get("torch_dtype")
if component_dtype is None or component_dtype == "auto":
component_dtype = default_torch_dtype
elif isinstance(component_dtype, str):
component_dtype = convert_dtype_str2torch(component_dtype.removeprefix("torch."))
torch_dtype[k] = component_dtype
pipe = pipelines.pipeline_utils.DiffusionPipeline.from_pretrained(
pretrained_model_name_or_path, torch_dtype=torch_dtype
)
pipe_config = pipe.load_config(pretrained_model_name_or_path)
elif isinstance(pretrained_model_name_or_path, pipelines.pipeline_utils.DiffusionPipeline) or (
modular_pipeline_cls is not None and isinstance(pretrained_model_name_or_path, modular_pipeline_cls)
):
pipe = pretrained_model_name_or_path
# a pipeline assembled in-process, as Modular Diffusers ones typically are,
# has no _name_or_path to reload the on-disk index from
name_or_path = pipe.config.get("_name_or_path", None)
pipe_config = pipe.load_config(name_or_path) if name_or_path is not None else {}
else:
raise ValueError(
f"Only support str, DiffusionPipeline or ModularPipeline class for model, "
f"but get {type(pretrained_model_name_or_path)}"
)
# add missing key
for k, v in pipe_config.items():
if k not in pipe.config:
pipe.config[k] = v
if hasattr(pipe, "unet"):
# Stable Diffusion pipelines (e.g., SD and SDXL) use a UNet denoiser.
model = pipe.unet
model_component_name = "unet"
else:
# DiT-based pipelines (e.g., Flux and SD3) use a Transformer denoiser.
model = pipe.transformer
model_component_name = "transformer"
# Diffusers keeps denoiser checkpoints below the pipeline repository root.
# Retain the component name so block-wise offloading can find those files.
model._autoround_checkpoint_subfolder = model_component_name
# Attach custom pipeline function for models that need special API calls
_attach_diffusion_pipeline_fn(pipe)
# meta model uses model.config.save_pretrained for config saving
model.config.save_pretrained = partial(config_save_pretrained, model.config, "config.json", model=model)
pipe.config.save_pretrained = partial(config_save_pretrained, pipe.config, "model_index.json")
def model_save_pretrained(model, save_directory, **kwargs):
super(model.__class__, model).save_pretrained(save_directory, **kwargs)
if hasattr(model.config, "quantization_config"):
model.config["quantization_config"] = model.config.quantization_config
with open(os.path.join(save_directory, "config.json"), "w", encoding="utf-8") as writer:
writer.write(json.dumps(dict(model.config), indent=2, sort_keys=True) + "\n")
# non-meta model uses model.save_pretrained for model and config saving
model.save_pretrained = partial(model_save_pretrained, model)
for comp_name in pipe.components:
comp = getattr(pipe, comp_name, None)
if (
comp_name.startswith("transformer")
and comp_name != "transformer"
and comp is not None
and isinstance(comp, torch.nn.Module)
):
comp._autoround_checkpoint_subfolder = comp_name
comp.config.save_pretrained = partial(config_save_pretrained, comp.config, "config.json", model=comp)
comp.save_pretrained = partial(model_save_pretrained, comp)
return pipe, model.to(device)
def load_model(
pretrained_model_name_or_path: str | torch.nn.Module,
platform: str = "hf",
model_dtype: str | None = None,
trust_remote_code: bool = True,
device: str = "cpu",
use_auto_mapping: bool | None = None,
use_model_replacements: bool = False,
**kwargs,
) -> tuple:
"""Unified model loader supporting LLM, MLLM (VLM), and diffusion models.
Automatically detects model type (text-only LLM / multimodal MLLM / diffusion
pipeline) via ``is_mllm_model`` / ``is_diffusion_model`` and dispatches to the
corresponding loader. When a pre-loaded ``torch.nn.Module`` is passed the
function skips loading and only performs type detection.
Args:
pretrained_model_name_or_path: HuggingFace model ID, local path, or an
already-loaded ``torch.nn.Module``.
platform: ``"hf"`` (default) or ``"model_scope"``.
model_dtype: Cast the model to this dtype after loading (e.g. ``"bf16"``).
trust_remote_code: Passed through to every HF ``from_pretrained`` call.
device: Device to load the model onto (default ``"cpu"``).
use_auto_mapping: Override ``device_map="auto"`` for MLLM/diffusion loaders.
``None`` means each sub-loader decides its own default.
use_model_replacements: Apply AutoRound replacement modules and special-model
forward patches after loading. Intended for isolated quantization workers.
**kwargs: Extra keyword arguments forwarded verbatim to the active sub-loader.
Returns:
A 7-tuple ``(model, tokenizer, processor, image_processor, pipe, is_mllm,
is_diffusion)`` where unused fields are ``None`` / ``False``.
"""
def _prepare_model(model):
if not use_model_replacements:
return model
from auto_round.special_model_handler import _handle_special_model, update_module
model = update_module(model, formats=None, cleanup_original=False)
return _handle_special_model(model)
model = None
tokenizer = None
processor = None
image_processor = None
pipe = None
_is_mllm = False
_is_diffusion = False
if isinstance(pretrained_model_name_or_path, str):
if is_mllm_model(pretrained_model_name_or_path, platform=platform):
_kwargs = dict(
platform=platform,
device=device,
model_dtype=model_dtype,
trust_remote_code=trust_remote_code,
)
if use_auto_mapping is not None:
_kwargs["use_auto_mapping"] = use_auto_mapping
_kwargs.update(kwargs)
model, processor, tokenizer, image_processor = mllm_load_model(pretrained_model_name_or_path, **_kwargs)
model = _prepare_model(model)
_is_mllm = True
elif is_diffusion_model(pretrained_model_name_or_path, trust_remote_code=trust_remote_code):
_kwargs = dict(
platform=platform,
device=device,
model_dtype=model_dtype,
trust_remote_code=trust_remote_code,
)
if use_auto_mapping is not None:
_kwargs["use_auto_mapping"] = use_auto_mapping
_kwargs.update(kwargs)
pipe, model = diffusion_load_model(pretrained_model_name_or_path, **_kwargs)
model = _prepare_model(model)
_is_diffusion = True
else:
# Plain text LLM
# Apply class-level block patches (e.g. unfused-MoE replacements) before
# ``from_pretrained`` so the model is loaded with the correct architecture.
from auto_round.modeling.unfused_moe import apply_model_monkey_patches
apply_model_monkey_patches(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
_kwargs = dict(
platform=platform,
device=device,
model_dtype=model_dtype,
trust_remote_code=trust_remote_code,
)
_kwargs.update(kwargs)
model, tokenizer = llm_load_model(pretrained_model_name_or_path, **_kwargs)
model = _prepare_model(model)
else:
# Already a loaded model object – detect type without triggering any file I/O.
try:
_is_mllm = is_mllm_model(pretrained_model_name_or_path)
except Exception:
pass
if not _is_mllm:
try:
_is_diffusion = is_diffusion_model(pretrained_model_name_or_path)
except Exception:
pass
model = _prepare_model(pretrained_model_name_or_path)
return model, tokenizer, processor, image_processor, pipe, _is_mllm, _is_diffusion
def is_pure_text_model(model):
"""verify on: phi-3.5, Mistral-Small-3.1, gemma-3, qwen2-vl,"""
if hasattr(model, "config") and hasattr(model.config, "vision_config"):
return False
if hasattr(model.__class__, "main_input_name") and model.__class__.main_input_name != "input_ids":
return False
for module in model.modules():
if hasattr(module.__class__, "main_input_name") and module.__class__.main_input_name != "input_ids":
return False
if "vision" in str(module.__class__).lower():
return False
if "image" in str(module.__class__).lower():
return False
if "img" in str(module.__class__).lower():
return False
return True
_is_mllm_model_cache: dict = {}
# Model types that have multimodal components but should use LLM compressor
# (text-only calibration, non-text modules excluded from quantization).
_LLM_ONLY_MODEL_TYPES = {"bagel"}
def get_model_name_or_path(model_or_path: str | torch.nn.Module) -> str | None:
if isinstance(model_or_path, str):
return model_or_path
return getattr(model_or_path, "_name_or_path", None) or getattr(model_or_path, "name_or_path", None)
_CODE_MODEL_TOKENS = {"code", "coder", "coding", "programming", "swe", "devstral"}
_CODE_MODEL_FAMILIES = {
"codellama",
"codegemma",
"codestral",
"deepseekcoder",
"granitecode",
"magicoder",
"opencoder",
"qwencoder",
"santacoder",
"stablecode",
"starcoder",
"wizardcoder",
}
_CODE_MODEL_TASKS = {"code-generation", "software-engineering", "text-to-code"}
def _match_code_model_name(value) -> str | None:
if not isinstance(value, str) or not value:
return None
value = re.split(r"[/\\]", value.rstrip("/\\"))[-1]
value = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", value)
token_matches = _CODE_MODEL_TOKENS.intersection(re.findall(r"[a-z]+", value.lower()))
if token_matches:
return min(token_matches)
components = re.findall(r"[a-z0-9]+", value.lower())
for family in sorted(_CODE_MODEL_FAMILIES):
if any(component == family or re.fullmatch(rf"{family}\d+", component) for component in components):
return family
return None
def _config_value(config, name: str):
if config is None:
return None
if isinstance(config, dict):
return config.get(name)
return getattr(config, name, None)
def _get_code_model_match(model_or_path, config=None):
config = config or getattr(model_or_path, "config", None)
candidates = [
("model name_or_path", get_model_name_or_path(model_or_path)),
("config._name_or_path", _config_value(config, "_name_or_path")),
("config.model_type", _config_value(config, "model_type")),
]
architectures = _config_value(config, "architectures") or []
if isinstance(architectures, str):
architectures = [architectures]
candidates.extend(("config.architectures", architecture) for architecture in architectures)
for source, value in candidates:
match = _match_code_model_name(value)
if match:
return source, match
task_candidates = []
for field in ("finetuning_task", "task", "pipeline_tag"):
task_candidates.append((f"config.{field}", _config_value(config, field)))
task_specific_params = _config_value(config, "task_specific_params") or {}
if isinstance(task_specific_params, dict):
task_candidates.extend(("config.task_specific_params", task) for task in task_specific_params)
for source, task in task_candidates:
if isinstance(task, str) and task.lower().replace("_", "-") in _CODE_MODEL_TASKS:
return source, task
return None
def is_code_model(model_or_path: str | torch.nn.Module, config=None) -> bool:
"""Return whether a pure-text model is explicitly specialized for code."""
match = _get_code_model_match(model_or_path, config)
if match is None:
return False
logger.info("Detected a code-specialized model from %s (matched %r).", *match)
return True
def is_mllm_model(model_or_path: str | torch.nn.Module, platform: str | None = None):
from auto_round.utils.common import MM_KEYS
model_path = get_model_name_or_path(model_or_path)
# Path-less in-process objects must be inspected independently rather than
# sharing a cache entry under None.
if model_path and model_path in _is_mllm_model_cache:
return _is_mllm_model_cache[model_path]
# Check model_type exclusion: some models have multimodal components
# but should be quantized as LLM (e.g., BAGEL MoT).
_model_type = None
if isinstance(model_or_path, torch.nn.Module) and hasattr(model_or_path, "config"):
_model_type = getattr(model_or_path.config, "model_type", None)
elif isinstance(model_path, str) and os.path.isdir(model_path):
_cfg_path = os.path.join(model_path, "config.json")
if os.path.exists(_cfg_path):
with open(_cfg_path) as _f:
_model_type = json.load(_f).get("model_type")
if _model_type in _LLM_ONLY_MODEL_TYPES:
return False
# For dummy model, model_path could be "".
# Only try to download if the path looks like a HF repo id (not a local filesystem path).
# Skip download for absolute paths or relative paths that contain current/parent dir markers.
# model_path is None for a model or pipeline built in-process, which has no name or path
_is_local_path = isinstance(model_path, str) and (os.path.isabs(model_path) or model_path.startswith(("./", "../")))
if model_path and not os.path.isdir(model_path) and not _is_local_path:
model_path = download_or_get_path(model_path, platform=platform)
result = False
if isinstance(model_path, str):
if os.path.exists(os.path.join(model_path, "preprocessor_config.json")) or os.path.exists(
os.path.join(model_path, "processor_config.json")
):
result = True
elif os.path.exists(os.path.join(model_path, "config.json")):
with open(os.path.join(model_path, "config.json")) as f:
config = json.load(f)
for key in config.keys():
if any(k in key for k in MM_KEYS):
result = True
break
if not result and isinstance(model_or_path, torch.nn.Module):
for name, module in model_or_path.named_modules():
if any(k in name for k in MM_KEYS):
result = True
break
# Cache by the original path key (model_path may have been resolved above)
original_key = get_model_name_or_path(model_or_path)
if original_key:
_is_mllm_model_cache[original_key] = result
return result
def is_gguf_model(model_path: str | torch.nn.Module) -> bool:
is_gguf_file = False
if isinstance(model_path, str):
if os.path.isfile(model_path) and model_path.endswith(".gguf"):
is_gguf_file = True
elif os.path.exists(model_path):
for file in os.listdir(model_path):
if file.endswith(".gguf"):
is_gguf_file = True
break
return is_gguf_file
## ModularPipeline.config_name, spelled out to keep the diffusers import lazy
MODULAR_PIPELINE_INDEX_NAME = "modular_model_index.json"
def _find_pipeline_index_file(model_dir_or_repo: str) -> str | None:
"""Return the pipeline index file of a diffusers directory or repo, if it has one.
Standard pipelines ship ``model_index.json``, Modular Diffusers pipelines ship
``modular_model_index.json`` instead.
"""
index_names = ("model_index.json", MODULAR_PIPELINE_INDEX_NAME)
if os.path.isdir(model_dir_or_repo):
for name in index_names:
index_file = os.path.join(model_dir_or_repo, name)
if os.path.exists(index_file):
check_diffusers_installed()
return index_file
return None
from huggingface_hub import hf_hub_download
for name in index_names:
try:
index_file = hf_hub_download(model_dir_or_repo, name)
check_diffusers_installed()
return index_file
except Exception as e:
logger.debug(f"No {name} found in {model_dir_or_repo}: {e}")
return None
def _get_modular_pipeline_class():
"""Return ModularPipeline when supported by the installed Diffusers version."""
try:
from diffusers.modular_pipelines import ModularPipeline
return ModularPipeline
except (ImportError, AttributeError):
return None
def is_diffusion_model(model_or_path: str | object, trust_remote_code: bool = True) -> bool:
from auto_round.utils.common import LazyImport
# Then check if model_index.json exists for diffusion pipeline,
# which is a strong signal of being a diffusion pipeline.
if isinstance(model_or_path, str):
# Quick check to avoid config loading attempts and unnecessary warnings
if is_gguf_model(model_or_path):
return False
# First check if it's a known diffusion pipeline by config/model_type
# to avoid unnecessary imports and file checks for non-diffusion models, which can be time-consuming.
try:
from transformers import AutoConfig
config = AutoConfig.from_pretrained(model_or_path, trust_remote_code=trust_remote_code)
model_type = getattr(config, "model_type", "")
# A special case for NextStep
if model_type == "nextstep":
return True
except Exception:
logger.warning(
f"Failed to load config for {model_or_path}, trying to check model_index.json for diffusion pipeline."
)
return _find_pipeline_index_file(model_or_path) is not None
elif not isinstance(model_or_path, torch.nn.Module):
check_diffusers_installed()
pipeline_utils = LazyImport("diffusers.pipelines.pipeline_utils")
if isinstance(model_or_path, pipeline_utils.DiffusionPipeline):
return True
modular_pipeline_cls = _get_modular_pipeline_class()
return modular_pipeline_cls is not None and isinstance(model_or_path, modular_pipeline_cls)
else:
return False
def detect_model_type(model):
"""Detect the type of model (LLM, MLLM, or Diffusion).
Args:
model: Model instance or model path string
Returns:
str: "mllm", "diffusion", or "llm"
"""
# Check if it's a diffusion model first (more specific)
if is_diffusion_model(model):
return "diffusion"
# Check if it's an MLLM
if is_mllm_model(model):
return "mllm"
# Default to standard LLM
return "llm"
def is_moe_layer(module: torch.nn.Module) -> bool:
"""Returns whether the module is an MOE layer."""
return "moe" in type(module).__name__.lower() or any(
key in type(module).__name__.lower()
for key in [
"MixtralSparseMoeBlock".lower(),
"ArcticMoE".lower(),
"DbrxFFN".lower(),
"MoELayer".lower(),
"PhimoeSparseMoeBlock".lower(),
"DeepseekMoE".lower(),
"DeepseekV2MoE".lower(),
"DeepseekV3MoE".lower(),
"Qwen2MoeSparseMoeBlock".lower(),
"Qwen3MoeSparseMoeBlock".lower(),
"Qwen3VLMoeTextSparseMoeBlock".lower(),
"Qwen3OmniMoeThinkerTextSparseMoeBlock".lower(),
"Qwen3OmniMoeTalkerTextSparseMoeBlock".lower(),
]
)
def get_block_names(model, quant_vision=False):
"""Get the block names for transformers-like networks.
Args:
model: The model.
Returns:
block_names: A list whose elements are list of block's layer names
"""
from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
def _search_block(name, module):
if hasattr(type(module), "__name__") and "ModuleList" in type(module).__name__:
return [(name, module)]
target_modules = []
for n, m in module.named_children():
if hasattr(type(m), "__name__") and "NgramEmbedding" in type(m).__name__:
continue
if hasattr(type(m), "__name__") and "ModuleList" in type(m).__name__:
target_modules.append((".".join(filter(None, (name, n))), m))
else:
target_modules.extend(_search_block(".".join(filter(None, (name, n))), m))
return target_modules
def _get_llm_block_names(model):
block_names = []
target_modules = _search_block("", model)
for i, target_m in enumerate(target_modules):
block_names.append([])
for n, m in target_m[1].named_children():
block_names[i].append(target_m[0] + "." + n)
return block_names
def _get_vlm_block_names(model, quant_vision=False, ignore_audio=True):
# Since calibration dataset doesn't contain audio data, audio-related blocks will be ignored by default.
effective_type = resolve_model_type(model)
if effective_type and effective_type in SPECIAL_MULTIMODAL_BLOCK:
return SPECIAL_MULTIMODAL_BLOCK[effective_type](model, quant_vision=quant_vision)
block_names = []
target_modules = []
target_modules = _search_block("", model)
for i, target_m in enumerate(target_modules):
if quant_vision or all(key not in target_m[0].lower() for key in VISION_MM_KEYS):
if ignore_audio and any(key in target_m[0].lower() for key in AUDIO_MM_KEYS):
continue
block_names.append([])
for n, m in target_m[1].named_children():
block_names[-1].append(target_m[0] + "." + n)
return block_names
# Check architecture-based special handlers first (e.g. MiMo-Audio has model_type="qwen2"
# but is_pure_text_model returns True since it has no vision modules — only audio ones).
effective_type = resolve_model_type(model)
if effective_type and effective_type in SPECIAL_MULTIMODAL_BLOCK:
return SPECIAL_MULTIMODAL_BLOCK[effective_type](model, quant_vision=quant_vision)
if quant_vision or not is_pure_text_model(model):
return _get_vlm_block_names(model, quant_vision=quant_vision)
else:
return _get_llm_block_names(model)
def get_lm_head_name(model):
block_names = get_block_names(model, True)
last_name = None
for n, m in model.named_modules():
if any(m.children()):
continue
last_name = n
for l in block_names:
if last_name in l:
last_name = None
break
return last_name
# please refer to https://github.com/NVIDIA/TensorRT-Model-Optimizer
# /blob/4c611e47a60084a86e1de7e48690a692a1b8170c/modelopt/torch/export/layer_utils.py#L976
def get_expert_linear_names(module: torch.nn.Module) -> list[str]:
"""Get the list of linear names for the experts."""
def module_match_name_list(module, name_list):
"""Check if the module name matches any of the names in the list.
e.g. module_match_name_list(QuantQwen3MoeSparseMoeBlock, ['Qwen3MoeSparseMoeBlock']) -> True
"""
return any(name.lower() in type(module).__name__.lower() for name in name_list)
if module_match_name_list(
module,
[
"Qwen2MoeSparseMoeBlock",
"Qwen3MoeSparseMoeBlock",
"DeepseekMoE",
"DeepseekV2MoE",
"DeepseekV3MoE",
"Qwen3VLMoeTextSparseMoeBlock",
"Qwen3OmniMoeThinkerTextSparseMoeBlock",
"Qwen3OmniMoeTalkerTextSparseMoeBlock",
],
):
return ["gate_proj", "down_proj", "up_proj"]
elif module_match_name_list(module, ["MixtralMoeSparseMoeBlock"]):
return ["linear_fc1", "linear_fc2"]
elif module_match_name_list(module, ["DBRXMoeSparseMoeBlock"]):
return ["w1_linear", "w2_linear", "v1_linear"]
else:
# assuming w1, w2, w3 by default
return ["w1", "w2", "w3"]
def get_expert_input_proj_names(module: torch.nn.Module) -> list[str]:
"""Get the list of input projection names for MoE experts.
Input projections are the first linear layers that receive the expert's input directly.
For FP8 dispatch efficiency, these projections need unified input scales across all experts.
Args:
module: The MoE module (e.g., SparseMoeBlock)
Returns:
List of input projection names (e.g., ['gate_proj', 'up_proj'])
"""
def module_match_name_list(module, name_list):
"""Check if the module name matches any of the names in the list."""
return any(name.lower() in type(module).__name__.lower() for name in name_list)
if module_match_name_list(
module,
[
"Qwen2MoeSparseMoeBlock",
"Qwen3MoeSparseMoeBlock",
"Qwen3VLMoeTextSparseMoeBlock",
"Qwen3OmniMoeThinkerTextSparseMoeBlock",
"Qwen3OmniMoeTalkerTextSparseMoeBlock",
"DeepseekMoE",
"DeepseekV2MoE",
"DeepseekV3MoE",
],
):
# gate_proj and up_proj are input projections, down_proj is output
return ["gate_proj", "up_proj"]
elif module_match_name_list(module, ["MixtralMoeSparseMoeBlock"]):
# Mixtral uses linear_fc1 as input projection, linear_fc2 is output
return ["linear_fc1"]
elif module_match_name_list(module, ["DBRXMoeSparseMoeBlock"]):
# w1_linear and v1_linear are input projections, w2_linear is output
return ["w1_linear", "v1_linear"]
else:
logger.warning_once("Using default input projection names ['w1', 'w3'] for MoE expert alignment. ")
# Default: w1 and w3 are input projections, w2 is output
return ["w1", "w3"]
def get_model_dtype(model_dtype, default="auto"):
if model_dtype is None or model_dtype == "auto":
model_dtype = default
elif model_dtype in ["bf16", "bfloat16"]:
model_dtype = "bfloat16"
elif model_dtype in ["f16", "float16", "fp16"]:
model_dtype = "float16"
elif model_dtype in ["f32", "float32", "fp32"]:
model_dtype = "float32"
else:
logger.warning(f"Unable to identify model_dtype {model_dtype}, reset to default model_dtype {default}")
model_dtype = default
return model_dtype
def get_nested_attr(module, attr_name: str):
"""Recursively get nested attribute (e.g., 'orig_layer.act_max')."""
attrs = attr_name.split(".")
for attr in attrs:
if not hasattr(module, attr):
return None
module = getattr(module, attr)
return module
def get_gguf_architecture(dir_model, model_type=ModelType.TEXT):
from auto_round.export.export_to_gguf.llama_cpp_conversion import get_conversion
is_mistral_format = False
if isinstance(dir_model, str):
dir_model = Path(dir_model)
conversion = get_conversion(dir_model, model_type=model_type)
hparams = conversion.ModelBase.load_hparams(dir_model, is_mistral_format)
if isinstance(hparams, dict):
tmp_model_type = hparams["model_type"]
else:
tmp_model_type = hparams.model_type
if "mistral" == tmp_model_type:
is_mistral_format = True
try:
hparams = conversion.ModelBase.load_hparams(dir_model, is_mistral_format)
except Exception:
is_mistral_format = False
if not is_mistral_format:
model_class = conversion.get_model_architecture(hparams, conversion.model_type(model_type))
elif model_type == ModelType.MMPROJ:
assert hparams.get("vision_encoder") is not None, "This model does not support multimodal"
model_class = "PixtralModel"
else:
model_class = "MistralModel"
return model_class
def get_layer_names_in_block(
model: torch.nn.Module,
supported_types=(torch.nn.Linear, transformers.pytorch_utils.Conv1D),
quant_block_list: list | None = None,
class_names: tuple | None = None,
) -> list[str]:
"""Retrieves the names of layers within each block of the model.
Returns:
list: A list of strings, where each string is the name of a layer
within a block of the model.
"""
if class_names is None:
class_names = []
for n, m in model.named_modules():
if type(m) in supported_types or (class_names is not None and m.__class__.__name__ in class_names):
m.bk_global_name = n
layers_in_block = []
if bool(quant_block_list):
all_blocks = quant_block_list
else:
all_blocks = get_block_names(model)
for block_names in all_blocks:
for block_name in block_names:
block = get_module(model, block_name)
for n, m in block.named_modules():
if hasattr(m, "bk_global_name"):
layers_in_block.append(m.bk_global_name)
delattr(m, "bk_global_name")
return layers_in_block
def set_nested_attr(module, attr_name: str, value):
"""Recursively set nested attribute (e.g., 'orig_layer.act_max' = value)."""
attrs = attr_name.split(".")
for attr in attrs[:-1]:
if not hasattr(module, attr):
return # No need to set act_max for fp layers
module = getattr(module, attr)
setattr(module, attrs[-1], value)
def check_to_quantized(config):
"""Checks if the configuration is valid for quantization.
Args:
config (dict or object): The configuration to check. It can be either a
dictionary with a 'bits' key or an object with a 'bits' attribute.
Returns:
bool: True if the configuration is valid for quantization (bits <= 8),
False otherwise.
"""
from auto_round.schemes import QuantizationScheme
if isinstance(config, (dict, QuantizationScheme)):
bits = config.get("bits", None)
act_bits = config.get("act_bits", None)
elif hasattr(config, "orig_layer"):
bits = getattr(config.orig_layer, "bits", None)
act_bits = getattr(config.orig_layer, "act_bits", None)
else:
bits = getattr(config, "bits", None)
act_bits = getattr(config, "act_bits", None)
bits = int(bits) if bits is not None else 16
act_bits = int(act_bits) if act_bits is not None else 16
return bits <= 8 or act_bits <= 8
def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):
"""
Check whether the input sequence length is within the limits defined
by the tokenizer and the model configuration.
Args:
input_seqlen (int): The length of the input sequence.
tokenizer: Optional, a HuggingFace tokenizer object.
model: Optional, a HuggingFace model object.
Returns:
ValueError: if the input length is not valid, riase Error.
"""
if model is not None and hasattr(model, "config"):
model_config = model.config
if hasattr(model_config, "max_position_embeddings") and input_seqlen > model_config.max_position_embeddings:
raise ValueError(
f"seqlen({input_seqlen}) exceeds model.config.max_position_embeddings("
f"{model_config.max_position_embeddings}). Please lowering '--seqlen'"
)
if tokenizer is not None and hasattr(tokenizer, "model_max_length") and input_seqlen > tokenizer.model_max_length:
raise ValueError(
f"seqlen({input_seqlen}) exceeds tokenizer.model_max_length({tokenizer.model_max_length}). "
"Please oncider Consider lowering the '--seqlen' or increasing tokenizer.model_max_length."
)
def cast_model_dtype(model: torch.nn.Module, dtype: torch.dtype) -> torch.nn.Module:
"""Cast a model without rounding its declared FP32 parameters and buffers."""
fp32_modules = set()
for attribute in ("_keep_in_fp32_modules", "_keep_in_fp32_modules_strict"):
names = getattr(model, attribute, None) or []
fp32_modules.update([names] if isinstance(names, str) else names)
if not fp32_modules:
return model.to(dtype)
# Inspect all aliases before casting so a shared tensor is protected even
# when its first name is outside the FP32 modules.
protected = set()
tensors = list(model.named_parameters(remove_duplicate=False)) + list(model.named_buffers(remove_duplicate=False))
for name, tensor in tensors:
if any(module_name in name for module_name in fp32_modules):
protected.add(id(tensor))
if isinstance(tensor, torch.nn.Parameter) and tensor.grad is not None:
protected.add(id(tensor.grad))
def convert(tensor):
if not (tensor.is_floating_point() or tensor.is_complex()):
return tensor
target_dtype = torch.float32 if id(tensor) in protected else dtype
return tensor.to(dtype=target_dtype)
return model._apply(convert)
def _to_model_dtype(model, model_dtype):
if isinstance(model_dtype, str):
try:
if (model_dtype == "float16" or model_dtype == "fp16") and model.dtype != torch.float16:
model = cast_model_dtype(model, torch.float16)
elif (
model_dtype == "bfloat16" or model_dtype == "bfp16" or model_dtype == "bf16"
) and model.dtype != torch.bfloat16:
model = cast_model_dtype(model, torch.bfloat16)
elif model_dtype == "float32" or model_dtype == "fp32":
model = cast_model_dtype(model, torch.float32)
except Exception:
logger.error("please use more device to fit the device or just use one device")
sys.exit()
return model
def get_attr(module, key):
"""Get attribute (including parameters like `...weight`) by dotted key.
Missing keys return `None` (legacy behavior relied on by tests).
"""
name_list = key.split(".")
for name in name_list:
if module is None:
return None
module = getattr(module, name, None)
return module
def set_attr(model, key, new_attr):
"""Set attribute (including parameters like `...weight`) by dotted key.
If an intermediate parent doesn't exist, this is a no-op.
"""
module = model
name_list = key.split(".")
for name in name_list[:-1]:
if not hasattr(module, name):
return
module = getattr(module, name)
setattr(module, name_list[-1], new_attr)
def get_module(module, key):
"""Get module from model by key name using PyTorch native API.
Missing paths return `None` to preserve legacy non-fail-fast behavior.
"""
try:
return module.get_submodule(key)
except (AttributeError, KeyError):
return None
def set_module(model, key, new_module):
"""Set new module into model by key name using PyTorch native API.
Missing paths are ignored (no-op) to preserve legacy behavior.
"""
try:
model.set_submodule(key, new_module)
except (AttributeError, KeyError):
return
def get_layer_features(layer):
"""Extracts input and output feature dimensions for supported layers."""
from auto_round.utils import deepspeed_exists
if deepspeed_exists:
from deepspeed.module_inject import LinearAllreduce, LinearLayer
if type(layer) == torch.nn.Linear:
return layer.in_features, layer.out_features
elif type(layer) == transformers.pytorch_utils.Conv1D: # TODO: Verify correctness
return layer.weight.shape[0], layer.weight.shape[1]
elif isinstance(layer, torch.nn.Embedding):
return layer.num_embeddings, layer.embedding_dim
elif deepspeed_exists and type(layer) in (LinearLayer, LinearAllreduce):
return layer.weight.shape[1], layer.weight.shape[0] # (input_dim, output_dim)
elif "FP8Linear" in layer.__class__.__name__:
return layer.in_features, layer.out_features
return None, None # Unsupported layer type
def get_common_prefix(paths):
# Split each path into components and find the common prefix
split_paths = [path.split(".") for path in paths]
common_prefix = split_paths[0]
for path in split_paths[1:]:
common_prefix = [comp for comp, other in zip(common_prefix, path) if comp == other]
return ".".join(common_prefix)
def unsupported_meta_device(model):
"""Checks if the model is a valid model for auto_round.
Args:
model: The model to be checked.
Returns:
bool: True if the model is valid, False otherwise.
"""
target_device = None
for param in model.parameters():
if target_device is None:
target_device = param.device
if param.device != target_device:
if param.device.type == "meta" or target_device.type == "meta":
return True
if target_device.type == "meta":
return not hasattr(model, "path")
return False
def map_nested_tensors(value: Any, transform: Callable[[torch.Tensor], torch.Tensor]) -> Any:
"""Apply ``transform`` to tensors in nested tuples, lists, and mappings."""
if torch.is_tensor(value):
return transform(value)
if isinstance(value, tuple):
return tuple(map_nested_tensors(item, transform) for item in value)
if isinstance(value, list):
return [map_nested_tensors(item, transform) for item in value]
if isinstance(value, (dict, UserDict)):
return type(value)({key: map_nested_tensors(item, transform) for key, item in value.items()})
return value
def to_device(input, device=torch.device("cpu")):
"""Moves input data to the specified device.
Args:
input: The input data to be moved.
device: The target device.
Returns:
The input data on the specified device.
"""
if input is None:
return None
if isinstance(input, torch.Tensor):
return input.to(device)
if isinstance(input, (dict, UserDict)):
for inp in input.keys():
input[inp] = to_device(input[inp], device)
elif isinstance(input, (list, tuple)):
if len(input) == 0:
return input
input_res = []
for inp in input:
input_res.append(to_device(inp, device))
if isinstance(input, tuple):
input_res = tuple(input_res)
input = input_res
return input
def move_to_device(input: Any, device=torch.device("cpu"), non_blocking: bool = False) -> Any:
"""Return a nested tensor container moved to *device* without mutating it."""
if input is None:
return None
if isinstance(input, torch.Tensor):
return input.to(device, non_blocking=non_blocking)
if isinstance(input, UserDict):
return type(input)({k: move_to_device(v, device, non_blocking) for k, v in input.items()})
if isinstance(input, dict):
return {k: move_to_device(v, device, non_blocking) for k, v in input.items()}
if isinstance(input, tuple):
return tuple(move_to_device(v, device, non_blocking) for v in input)
if isinstance(input, list):
return [move_to_device(v, device, non_blocking) for v in input]
return input
def mv_module_from_gpu(module):
"""Moves module from gpu to cpu.
Args:
module: The module to be moved.
Returns:
The module on the specified device.
"""
if hasattr(module, "device"):
if module.device.type in ("cpu", "meta"):
return module
has_meta = any(p.device.type == "meta" for p in module.parameters())
if not has_meta:
has_meta = any(b.device.type == "meta" for b in module.buffers())
if has_meta:
for _, child in module.named_children():
mv_module_from_gpu(child)
for attr_name in list(module._parameters.keys()):
p = module._parameters[attr_name]
if p is not None and p.device.type != "meta" and p.device.type != "cpu":
module._parameters[attr_name] = torch.nn.Parameter(p.to("cpu"), requires_grad=p.requires_grad)
for attr_name in list(module._buffers.keys()):
b = module._buffers[attr_name]
if b is not None and b.device.type != "meta" and b.device.type != "cpu":
module._buffers[attr_name] = b.to("cpu")
return module
return module.to("cpu")
def safe_device_move_with_meta_handling(
model,
target_device="cpu",
*,
materialize_meta=None,
logger=None,
):
"""Move model to target device, handling meta parameters and buffers correctly.
Skipping module move on target/meta device.
Args:
target_device: Target device ('cpu', 'cuda', etc.)
materialize_meta: Optional callable to materialize meta tensors before movement
logger: Optional logger for warnings if needed
"""
target_type = torch.device(target_device).type
# Materialize meta tensors if handler provided
if materialize_meta is not None:
materialize_meta(model)
meta_count = 0
# Move parameters
for p in model.parameters():
if p.device.type in (target_type, "meta"):
meta_count += p.device.type == "meta"
continue
p.data = p.data.to(target_device)
# Move buffers
for m in model.modules():
for n, b in m._buffers.items():
if b is None or b.device.type in (target_type, "meta"):
meta_count += b is not None and b.device.type == "meta"
continue
m._buffers[n] = b.to(target_device)
if meta_count and logger is not None:
logger.warning(f"{meta_count} tensors still on meta device after movement")
return model
def is_moe_model(model: torch.nn.Module) -> bool:
"""Heuristically detect whether ``model`` is a MoE architecture by scanning its config
for "moe"/"expert" markers or by checking module names for "expert".
"""
if hasattr(model, "config") and hasattr(model.config, "to_dict"):
config_str = str(model.config.to_dict()).lower()
if "moe" in config_str or "expert" in config_str:
return True
for n, m in model.named_modules():
if "expert" in n:
return True
return False
def is_moe_model_via_config(config) -> bool:
try:
config_str = str(config).lower()
except Exception:
config_str = str(config.to_dict()).lower() if hasattr(config, "to_dict") else ""
return "moe" in config_str or "expert" in config_str
def to_dtype(input, dtype=torch.float32):
"""Moves input data to the specified data type.
Args:
input: The input data to be moved.
dtype: The target data type.
Returns:
The input data on the specified data type.
"""
if input is None:
return None
if isinstance(input, torch.Tensor):
return input.to(dtype)
if isinstance(input, (dict, UserDict)):
for inp in input.keys():
input[inp] = to_dtype(input[inp], dtype)
elif isinstance(input, (list, tuple)):
if len(input) == 0:
return input
input_res = []
for inp in input:
input_res.append(to_dtype(inp, dtype))
if isinstance(input, tuple):
input_res = tuple(input_res)
input = input_res
return input
def set_amax_for_uncalibrated_experts(
experts: torch.nn.Module, set_amax_value: float | None = None, attr_name="act_max", unify_all: bool = False
):
"""Set amax of uncalibrated experts to a given value or the max of existing amax value from other experts.
Args:
experts: a list of experts
set_amax_value: set amax value to the given value.
If None, set amax value to the max of existing amax value from other experts.
attr_name: attribute name to set (default: "act_max")
unify_all: if True, unify the amax value for ALL experts (not just uncalibrated ones).
This is needed for FP8 dispatch where all experts must share the same input scale.
Returns:
uncalibrated_experts: a list of uncalibrated experts
"""
uncalibrated_experts = []
def _get_attr(module, name):
"""Get attribute from module or its orig_layer."""
if hasattr(module, name):
return getattr(module, name)
if hasattr(module, "orig_layer") and hasattr(module.orig_layer, name):
return getattr(module.orig_layer, name)
return None
def _get_amax_value(module):
value = get_nested_attr(module, attr_name)
if value is None and hasattr(module, "orig_layer"):
value = get_nested_attr(module.orig_layer, attr_name)
return value
# get the max amax value from all experts
if set_amax_value is None:
amax_values = [_get_amax_value(m) for m in experts if _get_amax_value(m) is not None]
if len(amax_values) == 0:
# Check if any expert actually needs act_max (act_bits < 8, not dynamic, not already quantized)
sample = next((m for m in experts if m is not None), None)
if sample is not None:
act_bits = _get_attr(sample, "act_bits")
act_dynamic = _get_attr(sample, "act_dynamic")
is_quantized = "Quant" in sample.__class__.__name__ or hasattr(sample, "is_mx")
needs_warning = (
not is_quantized and isinstance(act_bits, (int, float)) and act_bits < 8 and not act_dynamic
)
if needs_warning:
logger.warning_once(
f"All {len(experts)} expert layers are missing '{attr_name}' values. "
f"This may indicate calibration hooks were not attached to expert layers."
)
return uncalibrated_experts
# Flatten all tensors to 1D before concatenation
flat_values = [t.reshape(-1) for t in amax_values]
all_values = torch.cat(flat_values)
set_amax_value = torch.max(all_values)
set_amax_value = set_amax_value.unsqueeze(0) if set_amax_value.dim() == 0 else set_amax_value
for module in experts:
current_amax = _get_amax_value(module)
# Set amax if it's None (uncalibrated) OR if unify_all is True
if current_amax is None or unify_all:
# Use float32 dtype explicitly to ensure we create a floating point tensor
if not isinstance(set_amax_value, torch.Tensor):
set_amax_value = torch.tensor(set_amax_value, dtype=torch.float32)
set_nested_attr(module, attr_name, set_amax_value.clone())
if current_amax is None:
uncalibrated_experts.append(module)
if uncalibrated_experts:
logger.info_once(
f"Found {len(uncalibrated_experts)} uncalibrated expert layers. "
"Using max amax from calibrated experts to fill missing values. "
)
return uncalibrated_experts
# Please refer to: https://github.com/NVIDIA/TensorRT-Model-Optimizer/blob/
# 4c611e47a60084a86e1de7e48690a692a1b8170c/modelopt/torch/export/unified_export_hf.py#L195-L207
def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_name="act_max"):
if layer_name is not None:
parts = layer_name.split(".")
if "experts" not in parts:
raise ValueError
idx = parts.index("experts")
moe_name = ".".join(parts[:idx])
model = get_module(model, moe_name)
# Handle input quantizers of experts that are not calibrated
for name, sub_module in model.named_modules():
if not (is_moe_layer(sub_module) and hasattr(sub_module, "experts")):
continue
# Handle router (gate) layer - it's a Linear layer used for token routing
# It needs act_max for quantization but may not be calibrated if it wasn't exercised
_set_amax_for_moe_auxiliary_layers(sub_module, attr_name=attr_name)
expert_linear_names = get_expert_linear_names(sub_module)
# Get input projection names for FP8 dispatch unification
expert_input_proj_names = get_expert_input_proj_names(sub_module)
# Check experts structure and handle accordingly
if _is_fused_experts_module(sub_module.experts):
# Fused experts: 3D Parameters (e.g., DeepseekV2Experts)
# For fused experts, act_max is set on the parent MOE module, not individual experts
# Skip processing here as they don't have individual Linear layers to calibrate
logger.debug(
f"Skipping act_max setting for fused experts module '{name}': "
f"fused experts use parent module's act_max"
)
continue
elif isinstance(sub_module.experts, collections.abc.Iterable):
# Iterable experts: list of expert modules (e.g., Mixtral)
for linear_name in expert_linear_names:
try:
# Determine if this is an input projection that needs scale unification
unify_scale = linear_name in expert_input_proj_names and envs.AR_ENABLE_UNIFY_MOE_INPUT_SCALE
set_amax_for_uncalibrated_experts(
[getattr(expert, linear_name, None) for expert in sub_module.experts],
attr_name=attr_name,
unify_all=unify_scale, # Unify scales for input projections (gate/up)
)
except AttributeError as e:
# Provide more helpful debugging information
expert_types = list({type(expert).__name__ for expert in sub_module.experts})
raise AttributeError(
f"Failed to access attribute '{linear_name}' on experts. "
f"MoE module type: {type(sub_module).__name__}, "
f"Expert types: {expert_types}, "
f"Expected linear names: {expert_linear_names}. "
f"This suggests the get_expert_linear_names function may need "
f"to be updated for this model architecture. "
f"Original error: {e}"
) from e
else:
# Unknown experts structure
logger.warning(
f"Unknown experts structure in '{name}': type={type(sub_module.experts).__name__}. "
f"Skipping act_max setting. This may cause issues during export."
)
def _is_fused_experts_module(module: torch.nn.Module) -> bool:
"""Check if the module is a fused experts module (has 3D Parameter gate_up_proj/down_proj)."""
if not hasattr(module, "gate_up_proj") or not hasattr(module, "down_proj"):
return False
return (
isinstance(module.gate_up_proj, torch.nn.Parameter)
and isinstance(module.down_proj, torch.nn.Parameter)
and module.gate_up_proj.dim() == 3
and module.down_proj.dim() == 3
)
def _set_amax_for_moe_auxiliary_layers(moe_module: torch.nn.Module, attr_name: str = "act_max"):
"""Set amax for auxiliary layers in MOE modules (gate/router, shared_experts).
These layers are not part of the experts structure but are siblings in the MOE module.
They need act_max for quantization but may be missing if not all paths were exercised
during calibration.
Args:
moe_module: The MOE module (e.g., DeepseekV2MoE)
attr_name: The attribute name for amax (default: "act_max")
"""
# Collect all Linear layers that have act_bits set but missing act_max
layers_needing_amax = []
# Check gate (router) layer - it's typically a Linear layer for token routing
if hasattr(moe_module, "gate") and isinstance(moe_module.gate, torch.nn.Linear):
gate = moe_module.gate
if hasattr(gate, "act_bits") and gate.act_bits < 8:
if get_nested_attr(gate, attr_name) is None:
layers_needing_amax.append(gate)
# Check shared_experts - may have Linear layers that need act_max
if hasattr(moe_module, "shared_experts"):
shared_experts = moe_module.shared_experts
if shared_experts is not None:
for child_name, child in shared_experts.named_modules():
if isinstance(child, torch.nn.Linear):
if hasattr(child, "act_bits") and child.act_bits < 8:
if get_nested_attr(child, attr_name) is None:
layers_needing_amax.append(child)
if not layers_needing_amax:
return
# Try to get reference amax from calibrated experts
reference_amax = _get_reference_amax_from_experts(moe_module, attr_name)
if reference_amax is not None:
for layer in layers_needing_amax:
if not isinstance(reference_amax, torch.Tensor):
reference_amax = torch.tensor(reference_amax, dtype=torch.float32)
set_nested_attr(layer, attr_name, reference_amax.clone())
logger.info_once(
f"Set act_max for {len(layers_needing_amax)} MOE auxiliary layers (gate/shared_experts) "
f"using reference value from calibrated experts."
)
else:
logger.warning_once(
f"Cannot set act_max for {len(layers_needing_amax)} MOE auxiliary layers: "
f"no calibrated experts found to use as reference."
)
def _get_reference_amax_from_experts(moe_module: torch.nn.Module, attr_name: str = "act_max"):
"""Get a reference amax value from calibrated expert layers.
Args:
moe_module: The MOE module containing experts
attr_name: The attribute name for amax
Returns:
A reference amax tensor, or None if no calibrated experts found
"""
amax_values = []
if not hasattr(moe_module, "experts"):
return None
experts = moe_module.experts
# Handle iterable experts (list of modules)
if isinstance(experts, collections.abc.Iterable):
expert_linear_names = get_expert_linear_names(moe_module)
for expert in experts:
for linear_name in expert_linear_names:
layer = getattr(expert, linear_name, None)
if layer is not None:
amax = get_nested_attr(layer, attr_name)
if amax is not None:
amax_values.append(amax)
if not amax_values:
return None
# Return max of all amax values
flat_values = [t.reshape(-1) for t in amax_values]
all_values = torch.cat(flat_values)
return torch.max(all_values)
# Extra non-weight files that some models require at load time but are not saved
# by model.save_pretrained(). These are copied from the source model cache to
# the quantized output directory so that from_pretrained() works out of the box.
_EXTRA_MODEL_FILES = {
"spk_dict.pt", # Qwen2.5-Omni speaker dictionary for audio output
"llm_config.json", # BAGEL sub-model config
"vit_config.json", # BAGEL vision transformer config
"preprocessor_config.json", # BAGEL image preprocessor config
}
def _copy_extra_model_files(src_dir: str, dst_dir: str):
"""Copy known extra model files from *src_dir* to *dst_dir* if they exist."""
import os
import shutil
for file in os.listdir(src_dir):
if file in _EXTRA_MODEL_FILES:
src_file = os.path.join(src_dir, file)
dst_file = os.path.join(dst_dir, file)
if os.path.isfile(src_file) and not os.path.exists(dst_file):
logger.debug(f"Transferring extra model file {src_file} to {dst_dir}")
shutil.copy(src_file, dst_dir)
# Adapted from https://github.com/vllm-project/llm-compressor/blob/
# 5b3ddff74cae9651f24bef15d3255c4ee053fc60/src/llmcompressor/pytorch/model_load/helpers.py#L144
def copy_python_files_from_model_cache(model, save_path: str, copy_folders: bool | list[str] | tuple[str, ...] = False):
"""Copy Python files (and optionally subdirectories) from the model cache to *save_path*.
Args:
model: The model whose ``config._name_or_path`` points to the source cache.
save_path (str): Destination directory.
copy_folders (bool | list[str] | tuple[str, ...]): Controls which subdirectories
are copied from the cache root to *save_path*:
* ``False`` (default) – no folders are copied.
* ``True`` – every subdirectory that does not already exist in *save_path*
is copied (e.g. all of ``vae``, ``scheduler``, …).
* A list/tuple of folder names (e.g. ``["vae", "scheduler"]``) – only the
named subdirectories are copied.
"""
import shutil
from huggingface_hub import hf_hub_download
config = model.config
if not hasattr(config, "_name_or_path"):
return
if version.parse(transformers.__version__) < version.parse("5.0.0"):
from transformers.utils import TRANSFORMERS_CACHE
cache_dir = os.environ.get("HF_HOME", TRANSFORMERS_CACHE)
else:
from huggingface_hub.constants import HF_HUB_CACHE
cache_dir = os.environ.get("HF_HOME", HF_HUB_CACHE)
from transformers.utils import http_user_agent
cache_path = config._name_or_path
if not os.path.exists(cache_path):
user_agent = http_user_agent()
config_file_path = hf_hub_download(
repo_id=cache_path,
filename="config.json",
cache_dir=cache_dir,
force_download=False,
user_agent=user_agent,
)
cache_path = os.path.sep.join(config_file_path.split(os.path.sep)[:-1])
for file in os.listdir(cache_path):
full_file_name = os.path.join(cache_path, file)
if file.endswith(".py") and os.path.isfile(full_file_name):
logger.debug(f"Transferring {full_file_name} to {save_path}")
shutil.copy(full_file_name, save_path)
_copy_extra_model_files(cache_path, save_path)
if copy_folders is not False:
for entry in os.listdir(cache_path):
src_entry = os.path.join(cache_path, entry)
dst_entry = os.path.join(save_path, entry)
if not os.path.isdir(src_entry):
continue
if copy_folders is True or entry in copy_folders:
if not os.path.exists(dst_entry):
logger.debug(f"Transferring folder {src_entry} to {save_path}")
shutil.copytree(src_entry, dst_entry)
def extract_block_names_to_str(quant_block_list):
if not isinstance(quant_block_list, (list, tuple)):
return None
# Extract common prefix for each list
prefixes = [get_common_prefix(blocks) for blocks in quant_block_list]
# Join prefixes into a single string
return ",".join(prefixes)
def find_matching_blocks(model, all_blocks, to_quant_block_names):
"""
Find and return matching blocks in the model based on to_quant_block_names.
Args:
model: The model (not used in this specific function but kept for completeness).
all_blocks: List of lists, where each inner list contains full block names in the model.
to_quant_block_names: Comma-separated string of target block names to match.
Returns:
target_blocks: List of lists containing full paths of matching blocks in the model.
"""
if not to_quant_block_names:
return all_blocks
to_quant_block_list = to_quant_block_names
if isinstance(to_quant_block_names, (list, tuple)):
return to_quant_block_names
if isinstance(to_quant_block_names, str):
to_quant_block_list = [name.strip() for name in to_quant_block_names.split(",")]
target_blocks = []
for block_list in all_blocks:
matched_sublist = []
for name in to_quant_block_list:
matches = [block for block in block_list if re.search(name, block)]
if matches:
matched_sublist.extend(matches)
if matched_sublist:
target_blocks.append(matched_sublist)
if not target_blocks:
raise ValueError(
"No block names matched. Please check the input for to_quant_block_name,"
"or set to_quant_block_name to None to automatically match quantizable blocks."
)
return target_blocks
def is_separate_lm_head(model: torch.nn.Module) -> bool:
dir_path = model.name_or_path
if not os.path.isdir(dir_path):
dir_path = download_hf_model(dir_path)
lm_head_name: str = get_lm_head_name(model)
lm_head_name += ".weight"
if "model.safetensors.index.json" in os.listdir(dir_path):
with open(os.path.join(dir_path, "model.safetensors.index.json")) as f:
index_mapping = json.load(f)
return lm_head_name in index_mapping["weight_map"]
else:
from safetensors import safe_open
f = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")
return lm_head_name in f.keys()
def is_separate_tensor(model: torch.nn.Module, tensor_name: str) -> bool:
dir_path = model.name_or_path
if not os.path.isdir(dir_path):
dir_path = download_hf_model(dir_path)
if not tensor_name.endswith(".weight"):
tensor_name += ".weight"
if "model.safetensors.index.json" in os.listdir(dir_path):
with open(os.path.join(dir_path, "model.safetensors.index.json")) as f:
index_mapping = json.load(f)
return tensor_name in index_mapping["weight_map"]
else:
from safetensors import safe_open
f = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")
return tensor_name in f.keys()
def handle_generation_config(model: torch.nn.Module):
if hasattr(model, "generation_config"):
generation_config = model.generation_config
if hasattr(generation_config, "top_p") and generation_config.top_p != 1.0:
model.generation_config.do_sample = True
if hasattr(generation_config, "top_k") and generation_config.top_k != 0:
model.generation_config.do_sample = True
if hasattr(generation_config, "temperature") and generation_config.temperature != 1.0:
model.generation_config.do_sample = True
def merge_block_output_keys(block, input_others, extra_keys):
"""Merge block output keys into input_others, resolving positional/keyword conflicts.
When a block is called with positional args (stored in input_others["positional_inputs"]),
and the block output produces updated values for those same parameters (e.g.,
encoder_hidden_states), we must update the positional arg rather than adding a duplicate
keyword arg, which would cause "got multiple values for argument" errors.
"""
positional_inputs = input_others.get("positional_inputs")
if not positional_inputs or not extra_keys:
input_others.update(extra_keys)
return
try:
sig = inspect.signature(block.forward)
except (ValueError, TypeError):
input_others.update(extra_keys)
return
params = [p for p in sig.parameters.keys() if p != "self"]
# params[0] = hidden_states (passed as input_ids separately)
# params[1:] correspond to positional_inputs[0], [1], ...
positional_inputs = list(positional_inputs)
for key, value in extra_keys.items():
if key in params:
pos_idx = params.index(key) - 1 # -1 because hidden_states is params[0]
if 0 <= pos_idx < len(positional_inputs):
positional_inputs[pos_idx] = value
continue
input_others[key] = value
input_others["positional_inputs"] = tuple(positional_inputs)
def wrap_block_forward_positional_to_kwargs(base_hook):
"""Wrap a block forward hook to convert positional inputs to keyword args.
Models like GLM-Image call transformer blocks with positional args
(e.g. block(hidden_states, encoder_hidden_states, temb, ...)). The base
hook only stores positional_inputs once (from the first sample), losing
per-sample variation for encoder_hidden_states etc. By converting
positional args to keyword args, all inputs are properly accumulated
across calibration samples.
"""
_param_names_cache: dict = {}
def forward(m, hidden_states=None, *positional_inputs, **kwargs):
if positional_inputs:
m_id = id(m)
if m_id not in _param_names_cache:
# Prefer _true_orig_forward (set by new-arch CalibCompressor._replace_forward)
# over orig_forward (which points to the wrapped forward after wrapping).
sig_target = getattr(m, "_true_orig_forward", None) or m.orig_forward
sig = inspect.signature(sig_target)
_param_names_cache[m_id] = [p for p in sig.parameters.keys() if p != "self"]
_param_names = _param_names_cache[m_id]
for i, val in enumerate(positional_inputs):
param_idx = i + 1 # hidden_states is params[0]
if param_idx < len(_param_names):
param_name = _param_names[param_idx]
if param_name not in kwargs:
kwargs[param_name] = val
positional_inputs = ()
return base_hook(m, hidden_states, *positional_inputs, **kwargs)
return forward
def config_save_pretrained(config, file_name, save_directory, model=None):
if os.path.isfile(save_directory):
raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file")
os.makedirs(save_directory, exist_ok=True)
output_config_file = os.path.join(save_directory, file_name)
config_dict = dict(config)
if model is not None:
if file_name == "config.json" and hasattr(model.config, "quantization_config"):
config_dict["quantization_config"] = model.config.quantization_config
with open(output_config_file, "w", encoding="utf-8") as writer:
writer.write(json.dumps(config_dict, indent=2, sort_keys=True) + "\n")
def rename_weights_files(path: str, prefix="diffusion_pytorch_model"):
"""Rename weight files for diffusion models."""
import glob
import json
import os
# rename safetensors
files = sorted(glob.glob(f"{path}/*.safetensors"))
total = len(files)
if total == 1:
new = f"{prefix}.safetensors"
os.rename(files[0], os.path.join(path, new))
return
for i, f in enumerate(files, 1):
new = f"{prefix}-{i:05d}-of-{total:05d}.safetensors"
os.rename(f, os.path.join(path, new))
# rename index.json
idx = os.path.join(path, "model.safetensors.index.json")
if os.path.exists(idx):
with open(idx) as f:
d = json.load(f)
d["weight_map"] = {k: v.replace("model-", prefix + "-") for k, v in d["weight_map"].items()}
new_idx = os.path.join(path, f"{prefix}.safetensors.index.json")
with open(new_idx, "w") as f:
json.dump(d, f, indent=2)
os.remove(idx)
def _pin_module_execution_on_device(module: torch.nn.Module, device) -> None:
"""Attach an accelerate hook that runs ``module``'s forward on ``device``.
``AlignDevicesHook(execution_device=device, io_same_device=True)`` aligns the module's
inputs to ``device`` before the forward and moves the output back to the caller's device,
so a huge, non-quantizable embedding can stay resident on a chosen device (CPU host RAM,
or a specific accelerator) instead of being dragged around by generic block relocation.
Params are expected to already live on ``device`` (the caller moves them for GPU targets).
"""
from accelerate.hooks import AlignDevicesHook, add_hook_to_module
add_hook_to_module(module, AlignDevicesHook(io_same_device=True, execution_device=str(device)))
def _pin_module_execution_on_cpu(module: torch.nn.Module) -> None:
"""Attach an accelerate hook that keeps ``module``'s forward on CPU."""
_pin_module_execution_on_device(module, "cpu")
def module_pinned_execution_device(module: torch.nn.Module) -> str | None:
"""Return the execution-device string a module is pinned to, or ``None`` if not pinned.
Matches modules hooked by :func:`_pin_module_execution_on_device` (CPU or a specific
accelerator). Used by block relocation to leave intentionally-pinned modules in place.
"""
hook = getattr(module, "_hf_hook", None)
if hook is None:
return None
hooks = getattr(hook, "hooks", (hook,)) # SequentialHook aggregates several
for h in hooks:
exec_device = getattr(h, "execution_device", None)
if exec_device is not None:
return str(exec_device)
return None
def module_is_pinned_on_cpu(module: torch.nn.Module) -> bool:
"""Return True if ``module`` carries a CPU-execution accelerate hook."""
return module_pinned_execution_device(module) == "cpu"
def pin_ngram_embeddings_on_cpu_(module: torch.nn.Module) -> list:
"""Pin every ngram embedding found under ``module`` on CPU, in place.
Works on any subtree (a whole model *or* a single decoder block), so it can
be applied right after a block is materialized and before it is dispatched
onto accelerators. Returns the list of pinned module names.
"""
pinned = []
pinned_modules = []
for name, sub in module.named_modules():
leaf = name.rsplit(".", 1)[-1]
if leaf in ("ngram_embedding", "ngram_embeddings"):
if not module_is_pinned_on_cpu(sub):
_pin_module_execution_on_cpu(sub)
pinned.append(name)
pinned_modules.append(sub)
if pinned_modules:
_log_ngram_cpu_hint(pinned_modules)
return pinned
def _log_ngram_cpu_hint(ngram_modules: list) -> None:
"""Tell the user ngram embeddings are on CPU and how to move them (once per run).
Emits the total ngram size and, when ``AR_NGRAM_DEVICE`` is not explicitly set, points at
the faster on-GPU options -- noting that ``across`` (multi-GPU sharding) is experimental
and may have bugs.
"""
from auto_round import envs
total_nbytes = sum(_module_storage_nbytes(sub) for sub in ngram_modules)
if envs.is_set("AR_NGRAM_DEVICE"):
logger.info_once(
"Found %d ngram embedding module(s) (total %s); AR_NGRAM_DEVICE=%s keeps them on CPU.",
len(ngram_modules),
_format_nbytes_gib(total_nbytes),
str(getattr(envs, "AR_NGRAM_DEVICE", "auto")),
)
else:
logger.info_once(
"Found %d ngram embedding module(s) (total %s). AR_NGRAM_DEVICE is not set, so they stay on "
"CPU by default (memory-safe, but adds host<->device copies each block forward). To speed up, "
"set AR_NGRAM_DEVICE=<cuda:N|xpu:N> to place the whole table on one card, or AR_NGRAM_DEVICE=across "
"to row-shard it across all GPUs (experimental, may have bugs).",
len(ngram_modules),
_format_nbytes_gib(total_nbytes),
)
def move_to_device_preserving_cpu_pinned(module: torch.nn.Module, device) -> torch.nn.Module:
"""Move ``module`` to ``device`` but leave pinned/self-managed subtrees in place.
``nn.Module.to()`` recurses unconditionally and would drag a pinned child (e.g. a
multi-GiB ngram embedding pinned on CPU or on a specific accelerator) onto ``device``,
which is exactly what causes card-0 OOM on large ngram models. This walks the tree
manually and stops at any subtree pinned to an execution device
(:func:`_pin_module_execution_on_device`) or one that manages its own (possibly
multi-device) storage, e.g. a row-sharded :class:`_ShardedEmbedding`.
"""
if module_pinned_execution_device(module) is not None or _module_manages_own_device(module):
return module
# Materialize the iterators before mutating: reassigning ``param.data`` can trigger
# changes to the module's internal ``_parameters``/``_buffers`` dicts (e.g. on HPU lazy
# mode), which would raise ``RuntimeError: dictionary keys changed during iteration``.
for _, param in list(module.named_parameters(recurse=False)):
if param.device.type != "meta":
param.data = param.data.to(device)
if param.grad is not None:
param.grad.data = param.grad.data.to(device)
for _, buf in list(module.named_buffers(recurse=False)):
if buf.device.type != "meta":
buf.data = buf.data.to(device)
for child in list(module.children()):
move_to_device_preserving_cpu_pinned(child, device)
return module
class _ShardedEmbedding(torch.nn.Module):
"""Row-sharded, multi-GPU drop-in replacement for a huge ``nn.Embedding``.
A single embedding table ``(num_embeddings, dim)`` that is too large to fit on one
accelerator (e.g. the ~95 GiB Qwen4-Exp per-layer ngram table) is split along dim 0 into
contiguous row ranges, one shard per device. ``forward(ids)`` reproduces
``nn.Embedding(ids)`` bit-exactly by gathering each id from the shard that owns its row,
so the lookup runs on-GPU instead of being pinned (slowly) on CPU.
The module owns its (multi-device) storage: generic block-relocation helpers must skip it
(see :func:`_module_manages_own_device`), or the shards would be collapsed onto one device.
Shards are registered as non-persistent buffers so ``.to("cpu")`` based offload can still
reclaim their device memory after the block is done.
"""
def __init__(self, embedding: "torch.nn.Embedding", devices: list) -> None:
super().__init__()
weight = embedding.weight.data
self.num_embeddings, self.embedding_dim = int(weight.shape[0]), int(weight.shape[1])
self.weight_dtype = weight.dtype
n = max(1, len(devices))
rows_per_shard = (self.num_embeddings + n - 1) // n
self._bounds: list[tuple[int, int]] = []
for i, device in enumerate(devices):
lo = i * rows_per_shard
hi = min(self.num_embeddings, lo + rows_per_shard)
if lo >= hi:
break
self.register_buffer(f"shard_{i}", weight[lo:hi].to(device), persistent=False)
self._bounds.append((lo, hi))
@property
def weight(self):
# Exposed so callers that read ``embedding.weight.device`` keep working (the forward
# re-routes ids to the correct shard regardless of which device they arrive on).
return self.shard_0
def forward(self, input_ids: "torch.Tensor") -> "torch.Tensor":
flat = input_ids.reshape(-1)
# Compile-friendly gather: no data-dependent control flow. Avoid ``mask.any()`` (a scalar
# sync -> dynamo graph break) and boolean-mask indexing (``flat[mask]`` -> data-dependent
# shapes -> recompiles). Instead, for every shard clamp ids into its local range, gather,
# and select the owning shard's rows with ``torch.where``. Each id lives in exactly one
# shard, so exactly one iteration writes it; the rest keep the running ``out``.
out = torch.zeros(flat.shape[0], self.embedding_dim, dtype=self.weight_dtype, device=flat.device)
for i, (lo, hi) in enumerate(self._bounds):
shard = getattr(self, f"shard_{i}")
mask = (flat >= lo) & (flat < hi)
local = torch.clamp(flat - lo, 0, hi - lo - 1).to(shard.device)
gathered = torch.nn.functional.embedding(local, shard).to(out.device)
out = torch.where(mask.unsqueeze(-1), gathered, out)
return out.reshape(*input_ids.shape, self.embedding_dim)
def _module_manages_own_device(module: torch.nn.Module) -> bool:
"""True for modules that place their own (possibly multi-device) storage.
Such modules (e.g. :class:`_ShardedEmbedding`) must be skipped by the generic block
relocation helpers so their shards are not collapsed onto a single device.
"""
return isinstance(module, _ShardedEmbedding)
def shard_ngram_embeddings_across_gpus_(module: torch.nn.Module, devices: list) -> list:
"""Row-shard every plain ``nn.Embedding`` ngram table under ``module`` across ``devices``.
Requires >= 2 devices; only plain, already-materialized ``nn.Embedding`` tables are
sharded (anything else is left for the caller to CPU-pin). Returns the sharded names.
"""
if not devices or len(devices) < 2:
return []
sharded = []
for name, sub in list(module.named_modules()):
leaf = name.rsplit(".", 1)[-1]
if leaf not in ("ngram_embedding", "ngram_embeddings"):
continue
if not isinstance(sub, torch.nn.Embedding) or isinstance(sub, _ShardedEmbedding):
continue
if sub.weight.device.type == "meta":
continue # not materialized yet; caller decides
parent = get_module(module, name.rsplit(".", 1)[0]) if "." in name else module
try:
replacement = _ShardedEmbedding(sub, devices)
except Exception as err: # OOM or device error -> leave for the caller to CPU-pin
logger.warning(f"Could not shard ngram embedding '{name}' across GPUs ({err}); keeping it on CPU.")
continue
setattr(parent, leaf, replacement)
sharded.append(name)
return sharded
def _iter_ngram_modules(module: torch.nn.Module):
"""Yield ``(name, submodule)`` for every ngram embedding leaf under ``module``."""
for name, sub in module.named_modules():
leaf = name.rsplit(".", 1)[-1]
# Keep exact matches first, but also accept common ngram leaf variants.
if leaf in ("ngram_embedding", "ngram_embeddings") or ("ngram_embedding" in leaf or leaf.startswith("ngram")):
yield name, sub
def _module_storage_nbytes(module: torch.nn.Module) -> int:
"""Return parameter+buffer storage bytes for one module (including its children)."""
total = 0
for param in module.parameters(recurse=True):
total += int(param.numel()) * int(param.element_size())
for buf in module.buffers(recurse=True):
total += int(buf.numel()) * int(buf.element_size())
return total
def _format_nbytes_gib(nbytes: int) -> str:
"""Format byte count in GiB for user-facing logs."""
gib = float(nbytes) / (1024.0**3)
return f"{gib:.2f} GiB"
def _place_ngram_on_single_device_(sub: torch.nn.Module, device: str) -> None:
"""Keep one ngram embedding resident on ``device`` (CPU or a specific accelerator).
For a GPU target the params are physically moved onto the card first (so the on-device
lookup is real); for CPU they stay in host RAM. An ``AlignDevicesHook`` then keeps the
module executing on ``device`` and re-aligns I/O to the caller, and marks it so the
generic block relocation leaves it in place.
"""
if module_pinned_execution_device(sub) == str(device):
return
if not isinstance(sub, _ShardedEmbedding):
try:
tensors = list(sub.parameters(recurse=False)) + list(sub.buffers(recurse=False))
if any(t.device.type != "meta" for t in tensors):
sub.to(device)
except Exception as err: # OOM, invalid/unavailable device, ... -> safe CPU fallback
logger.warning(f"Could not place ngram embedding on {device} ({err}); keeping it on CPU.")
device = "cpu"
_pin_module_execution_on_device(sub, device)
def _normalize_ngram_target_device(setting: str, devices: list) -> str | None:
"""Resolve an ``AR_NGRAM_DEVICE`` value naming a single device to a concrete device string.
Accepts a full string (``cuda:1``/``xpu:0``/``cpu``) or a bare index (``1`` -> the second
entry of ``devices`` when present, else ``<accelerator>:1``). Returns ``None`` if the value
is not a single-device request.
"""
if setting == "cpu":
return "cpu"
if ":" in setting:
return setting
if setting.isdigit():
idx = int(setting)
if devices and idx < len(devices):
return str(devices[idx])
try:
from auto_round.utils.device_manager import device_manager
return f"{device_manager.type}:{idx}"
except Exception: # pragma: no cover - device manager optional/unavailable
return f"cuda:{idx}"
return None
def place_ngram_embeddings_for_tuning_(module: torch.nn.Module, gpu_devices: list | None = None) -> list:
"""Place ngram embeddings for per-block tuning, honoring the ``AR_NGRAM_DEVICE`` env var.
``AR_NGRAM_DEVICE`` values:
* ``auto`` (default): keep the table pinned on CPU (memory-safe; the table does not
participate in tuning). A one-time hint points users at the faster on-GPU options.
* ``across`` / ``shard`` / ``gpu``: force row-sharding across all available GPUs.
* ``cpu``: pin on CPU (same as the default).
* a device (``cuda:1``, ``xpu:0``, or a bare index like ``1``): put the whole table on
that specific card.
Multi-GPU sharding is experimental (see the warning below). Returns the handled names.
"""
from auto_round import envs
setting = str(getattr(envs, "AR_NGRAM_DEVICE", "auto")).strip().lower()
ngram_device_explicitly_set = bool(envs.is_set("AR_NGRAM_DEVICE"))
devices = [str(d) for d in gpu_devices] if gpu_devices else []
if not devices:
try:
from auto_round.utils.device_manager import device_manager
if device_manager.is_available() and device_manager.type != "cpu":
devices = [str(d) for d in device_manager.device_list]
except Exception: # pragma: no cover - device manager optional/unavailable
devices = []
ngram_modules = list(_iter_ngram_modules(module))
if not ngram_modules:
fallback_ngram_names = [name for name, _ in module.named_modules() if "ngram" in name.lower()]
if fallback_ngram_names:
logger.info_once(
"AR_NGRAM_DEVICE=%s is set, but no ngram embedding leaf matched placement rules in this block. "
"Found %d ngram-like module name(s), e.g. %s",
setting or "auto",
len(fallback_ngram_names),
", ".join(fallback_ngram_names[:3]),
)
return []
total_ngram_nbytes = sum(_module_storage_nbytes(sub) for _, sub in ngram_modules)
logger.info_once(
"Detected %d ngram embedding module(s), total size %s (AR_NGRAM_DEVICE=%s).",
len(ngram_modules),
_format_nbytes_gib(total_ngram_nbytes),
setting or "auto",
)
# Resolve the requested mode.
single_target = _normalize_ngram_target_device(setting, devices)
if setting in ("across", "shard", "gpu"):
mode = "across"
elif setting in ("auto", ""):
# Default: keep the (huge, non-tunable) ngram table on CPU. It does not participate in
# tuning, so host residency is correct and memory-safe (no per-card OOM, no extra device
# RAM churn); the only cost is host<->device copies per forward. Point users at the
# faster on-GPU options once, so they can opt in when they have the headroom.
mode = "single"
single_target = "cpu"
if not ngram_device_explicitly_set:
logger.info_once(
"AR_NGRAM_DEVICE is not set, so ngram embeddings stay on CPU by default (total %s, memory-safe). "
"For faster lookup set AR_NGRAM_DEVICE=<cuda:N|xpu:N> (single GPU) or AR_NGRAM_DEVICE=across "
"(multi-GPU sharding, experimental and may have bugs).",
_format_nbytes_gib(total_ngram_nbytes),
)
else:
logger.info_once(
"AR_NGRAM_DEVICE=%s keeps ngram embeddings on CPU (total %s, memory-safe).",
setting,
_format_nbytes_gib(total_ngram_nbytes),
)
elif single_target is not None:
mode = "single"
else:
logger.warning(f"Unrecognized AR_NGRAM_DEVICE={setting!r}; falling back to CPU (safe default).")
mode = "single"
single_target = "cpu"
handled: list = []
if mode == "across":
if len(devices) >= 2:
logger.warning_once(
"Sharding ngram embeddings across multiple GPUs is experimental and may have bugs; "
"set AR_NGRAM_DEVICE=cpu (safe) or a specific card (e.g. AR_NGRAM_DEVICE=cuda:0) if you hit issues."
)
handled = list(shard_ngram_embeddings_across_gpus_(module, devices))
if handled:
logger.info(
"Placed %d ngram embedding module(s) across %d GPUs (total %s).",
len(handled),
len(devices),
_format_nbytes_gib(total_ngram_nbytes),
)
# Any ngram not sharded (single/no GPU, custom module, OOM) falls back to CPU pin below.
fallback = devices[0] if len(devices) == 1 else "cpu"
for name, sub in ngram_modules:
if name in handled or isinstance(sub, _ShardedEmbedding):
continue
_place_ngram_on_single_device_(sub, fallback)
handled.append(name)
if handled and len(devices) < 2:
logger.info(
"Placed %d ngram embedding module(s) on %s (total %s).",
len(handled),
fallback,
_format_nbytes_gib(total_ngram_nbytes),
)
return handled
# mode == "single": every ngram table on one chosen device (CPU or a specific card).
target = single_target if single_target is not None else "cpu"
for name, sub in ngram_modules:
_place_ngram_on_single_device_(sub, target)
handled.append(name)
if handled:
logger.info(
"Placed %d ngram embedding module(s) on %s (total %s).",
len(handled),
target,
_format_nbytes_gib(total_ngram_nbytes),
)
return handled
def hook_ngram_embeddings_on_cpu(model):
"""Pin ngram embeddings on CPU so they are never moved onto an accelerator.
Handles two layouts:
* The top-level ``model.model.ngram_embeddings`` module (original behavior).
* Per-layer ngram embeddings nested anywhere in the tree, e.g.
``model.language_model.layers.N.ple.ple_embedding.ngram_embedding`` used by
Qwen3-Next-Flash style checkpoints. These are single, very large lookup
tables (~tens of GiB) that do not participate in tuning, so keeping them on
CPU avoids GPU OOM during per-block dispatch.
Returns ``(has_top_level_ngram, raw_top_level_ngram_or_None)`` for backward
compatibility with the calibration dispatch restore logic.
"""
# Per-layer / nested ngram embeddings (match any module attribute literally
# named ``ngram_embedding``, plus the pluralized top-level ``ngram_embeddings``).
pin_ngram_embeddings_on_cpu_(model)
has_ngram_embeddings = hasattr(model, "model") and hasattr(model.model, "ngram_embeddings")
raw_ngram_embeddings = model.model.ngram_embeddings if has_ngram_embeddings else None
return has_ngram_embeddings, raw_ngram_embeddings
def is_model_free_route(
model,
scheme,
iters: int,
disable_opt_rtn,
kwargs: dict,
) -> bool:
"""Return True when the model-free fast-path should be taken.
Mirrors the ``is_diffusion_model`` / ``is_mllm_model`` helpers used in
``AutoRound.__new__`` to select the right compressor class.
Model-free mode is activated when **either** of the following holds:
* ``model_free=True`` is explicitly set in *kwargs*.
* All of the following are true:
- ``disable_model_free`` is not set (or False) in *kwargs*
- *model* is a string (HF hub ID or local path)
- *iters* == 0
- *disable_opt_rtn* is exactly ``True``
- *scheme* is a supported model-free preset
Note: this function only *reads* kwargs; it does **not** pop any keys.
"""
from auto_round.compressors.model_free import (
_looks_like_auto_scheme,
_validate_auto_scheme_options,
is_model_free_supported_scheme,
)
explicit = bool(kwargs.get("model_free", False))
disabled = bool(kwargs.get("disable_model_free", False))
has_static_attention_quantization = any(
kwargs.get(key) is not None for key in ("static_kv_dtype", "static_attention_dtype")
)
if explicit:
if has_static_attention_quantization:
raise ValueError("model_free=True does not support static KV or attention quantization.")
return True
if has_static_attention_quantization:
return False
# Only auto-route when format is auto_round (or not specified).
fmt = kwargs.get("format", "auto_round")
if fmt is None:
fmt = "auto_round"
fmt_first = str(fmt).lower().replace(" ", "").split(",")[0]
common_conditions = not disabled and isinstance(model, str) and iters == 0 and disable_opt_rtn is True
if _looks_like_auto_scheme(scheme):
try:
family = _validate_auto_scheme_options(scheme)
except ValueError:
return False
if fmt_first == "auto_round":
return common_conditions and family == "int"
if fmt_first == "llm_compressor":
return common_conditions and family == "mx_fp"
return False
if fmt_first == "llm_compressor":
from auto_round.compressors.model_free import _apply_scheme_overrides
from auto_round.schemes import is_mx_fp as _is_mx_fp
try:
scheme_obj = _apply_scheme_overrides(scheme, kwargs)
scheme_is_mx_fp = _is_mx_fp(scheme_obj.data_type or "")
except Exception:
scheme_is_mx_fp = False
return common_conditions and scheme_is_mx_fp and is_model_free_supported_scheme(scheme, kwargs)
if fmt_first != "auto_round":
return False
return common_conditions and is_model_free_supported_scheme(scheme, kwargs)
def find_layers_from_config(model_dir: str, class_names: list[str] | None = None) -> dict[str, str]:
"""Detect layers of given class names by loading the model on ``device='meta'``.
Only ``config.json`` is required — no weights are read.
For regular models the root directory is checked. For diffusion-style
repos (no root ``config.json`` but a ``transformer/`` subfolder), only the
``transformer/`` subfolder is checked — other sub-components (``vae/``,
``scheduler/``, …) are intentionally skipped because only the transformer
is quantized in model-free mode.
Args:
model_dir: Local directory containing ``config.json``, or a diffusion
repo root whose ``transformer/`` subfolder contains ``config.json``.
class_names: Class names to look for, matched against
``type(module).__name__``. Defaults to
``["Embedding", "Conv1d", "Conv1D"]`` — the types incompatible
with model-free RTN packing.
Returns:
``{class_name: [layer_name, ...]}`` for every matched module.
Returns an empty dict on any failure.
"""
from huggingface_hub import snapshot_download
from transformers import AutoConfig, AutoModel
if class_names is None:
class_names = ["Embedding", "Conv1d", "Conv1D"]
if isinstance(class_names, str):
class_names = [class_names]
target = set(class_names)
# download if not local, but only the config files (fast)
if not os.path.exists(model_dir):
model_dir = snapshot_download(
repo_id=model_dir,
allow_patterns=["**/config.json"],
)
# Build the list of (prefix, config_dir) pairs to inspect.
# For diffusion repos (no root config.json) only check transformer/.
# For regular repos only check the root directory.
dirs: list[tuple[str, str]] = []
if os.path.exists(os.path.join(model_dir, "config.json")):
dirs.append(("", model_dir))
else:
transformer_dir = os.path.join(model_dir, "transformer")
if os.path.isdir(transformer_dir) and os.path.exists(os.path.join(transformer_dir, "config.json")):
dirs.append(("", transformer_dir))
result: dict[str, str] = {}
for prefix, config_dir in dirs:
try:
with torch.device("meta"):
config = AutoConfig.from_pretrained(config_dir, trust_remote_code=True)
model = AutoModel.from_config(config, trust_remote_code=True)
except Exception as e:
logger.warning(f"Failed to load model from {config_dir} for layer detection. Skipping. Warning: {e}")
continue # skip silently
for name, module in model.named_modules():
cls_name = type(module).__name__
if any(t.lower() in cls_name.lower() for t in target):
full_name = f"{prefix}.{name}" if prefix else name
if cls_name not in result:
result[cls_name] = [full_name]
else:
result[cls_name].append(full_name)
del model
return result