Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

Support ByteDance-Seed/BAGEL-7B-MoT quantization in w4a16 format (#1633)

Liang Lv91679a5c30ea
diff --git a/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt b/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt
index 41b7edde1..c829fc71b 100644
--- a/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt
+++ b/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt
@@ -1 +1,3 @@
-endianess
\ No newline at end of file
+endianess
+MOT
+MoT
\ No newline at end of file
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index d70010dc1..65e2158b2 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -19,7 +19,7 @@ from dataclasses import asdict, dataclass, fields
from typing import Any, Optional, Union
import torch
-from transformers import set_seed
+from transformers import AutoConfig, set_seed
from auto_round.algorithms.alg_config import AlgConfig
from auto_round.algorithms.quantization import BaseQuantizers, QuantizationConfig
@@ -127,6 +127,22 @@ class BaseCompressor(object):
_scheme_resolved: bool = False
scheme_generator = None
+ @staticmethod
+ def _preload_model_config(model: Union[torch.nn.Module, str], trust_remote_code: bool) -> Optional[AutoConfig]:
+ if not isinstance(model, str):
+ return None
+
+ try:
+ return AutoConfig.from_pretrained(model, trust_remote_code=trust_remote_code)
+ except (OSError, EnvironmentError, ValueError) as e:
+ logger.debug(
+ "Failed to load config via AutoConfig.from_pretrained for %s: %s. "
+ "Proceeding without config-based checks.",
+ model,
+ e,
+ )
+ return None
+
def __init__(
self,
config: Union[AlgConfig, list[AlgConfig]],
@@ -273,6 +289,7 @@ class BaseCompressor(object):
# allocation early in the heap, matching the OLD arch allocation order
# and reducing C-heap fragmentation (which is amplified on HPU).
_device = get_major_device(device_map if device_map is not None else 0)
+ model_config = self._preload_model_config(model, trust_remote_code)
self.model_context = ModelContext(
model,
@@ -280,6 +297,7 @@ class BaseCompressor(object):
platform=platform,
model_dtype=model_dtype,
trust_remote_code=trust_remote_code,
+ config=model_config,
amp=amp,
need_calib=self.need_calib,
device=_device,
@@ -739,6 +757,10 @@ class BaseCompressor(object):
self.ignore_layers = cfg.ignore_layers
self.quant_lm_head = cfg.quant_lm_head
self.to_quant_block_names = cfg.to_quant_block_names
+ if self.to_quant_block_names is None:
+ self.to_quant_block_names = getattr(self.model_context.model, "_autoround_to_quant_block_names", None)
+ if self.to_quant_block_names is not None:
+ self.quantize_config.to_quant_block_names = self.to_quant_block_names
# Resolve the scheme (pure config work: sets data_type / bits / sym /
# scale_dtype etc. on both self and self.quantize_config).
diff --git a/auto_round/context/model.py b/auto_round/context/model.py
index 9bb46152c..a29db4073 100644
--- a/auto_round/context/model.py
+++ b/auto_round/context/model.py
@@ -62,6 +62,7 @@ class ModelContext(BaseContext):
platform="hf",
model_dtype=None,
trust_remote_code=True,
+ config: Optional[AutoConfig] = None,
amp=True,
need_calib=True,
device="cpu",
@@ -92,6 +93,7 @@ class ModelContext(BaseContext):
self.platform = platform
self.model_dtype = model_dtype
self.trust_remote_code = trust_remote_code
+ self.config = config
self.amp = amp
self.need_calib = need_calib
self.quant_nontext_module = quant_nontext_module
@@ -139,11 +141,12 @@ class ModelContext(BaseContext):
self.model, platform=self.platform, device="cpu", model_dtype=self.model_dtype
)
elif isinstance(self.model, str):
- config: Optional[AutoConfig] = None
+ config = self.config
try:
- config = AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)
+ if config is None:
+ config = AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)
self._import_custom_moe_replacements(config)
- except (OSError, EnvironmentError) as e:
+ except (OSError, EnvironmentError, ValueError) as e:
logger.debug(
"Failed to load config via AutoConfig.from_pretrained for %s: %s. "
"Proceeding without config-based checks.",
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 264c77ab8..6f11b454c 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -49,6 +49,7 @@ SUPPORT_ONLY_TEXT_MODELS = [
"qwen3_omni_moe",
"gemma3",
"gemma4",
+ "bagel",
]
NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]
@@ -336,11 +337,35 @@ def _get_glm_image_multimodal_block(model, quant_vision=False):
return block_names
+def _get_bagel_multimodal_block(model, quant_vision=False):
+ """Get block names for BAGEL MoT (Mixture of Transformers) model.
+
+ BAGEL model structure:
+ - language_model.model.layers: Qwen2-based LLM with MoT dual paths
+ - vit_model: SigLIP vision encoder (not quantized by default)
+ - connector: Vision-language MLP connector
+ - encoder/decoder: VAE autoencoder
+ - time_embedder, vae2llm, llm2vae: bridge modules
+
+ By default, only the language_model layers are quantized.
+ """
+ block_names = []
+
+ if hasattr(model, "language_model") and hasattr(model.language_model, "model"):
+ if hasattr(model.language_model.model, "layers"):
+ block_names.append(
+ [f"language_model.model.layers.{i}" for i in range(len(model.language_model.model.layers))]
+ )
+
+ return block_names
+
+
SPECIAL_MULTIMODAL_BLOCK = {
"deepseek_vl_v2": _get_deepseek_vl2_multimodal_block,
"qwen2_5_omni": _get_qwen2_5_omni_multimodal_block,
"qwen3_omni_moe": _get_qwen3_omni_moe_multimodal_block,
"glm_image": _get_glm_image_multimodal_block,
+ "bagel": _get_bagel_multimodal_block,
}
@@ -721,6 +746,46 @@ register_ignore_layers(
)
+def get_bagel_ignore_layers(model) -> list[str]:
+ """Keep BAGEL generation-path modules in FP16.
+
+ BAGEL uses `*_moe_gen` modules for the image-generation path. Quantizing
+ them causes quality to collapse during the iterative denoising loop.
+ The shared attention projections are also highly sensitive, and preserving
+ the top 4 transformer blocks in FP16 gave acceptable image quality in
+ validation runs.
+ """
+ top_fp16_layers = 0
+
+ ignore_layers = [
+ "moe_gen",
+ "self_attn.q_proj",
+ "self_attn.k_proj",
+ "self_attn.v_proj",
+ "self_attn.o_proj",
+ ]
+
+ num_layers = 0
+ if hasattr(model, "language_model") and hasattr(model.language_model, "model"):
+ num_layers = len(getattr(model.language_model.model, "layers", []))
+
+ if num_layers > 0:
+ for layer_idx in range(max(0, num_layers - top_fp16_layers), num_layers):
+ ignore_layers.append(f"language_model.model.layers.{layer_idx}")
+
+ return ignore_layers
+
+
+register_ignore_layers(
+ matchers=[
+ ModelTypeMatcher(r"bagel", mode="full"),
+ ],
+ ignore_layers=[
+ get_bagel_ignore_layers,
+ ],
+)
+
+
def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:
layers = []
for rule in _PRE_DEFINED_IGNORE_LAYERS:
diff --git a/auto_round/utils/bagel_loader.py b/auto_round/utils/bagel_loader.py
new file mode 100644
index 000000000..a4fb42599
--- /dev/null
+++ b/auto_round/utils/bagel_loader.py
@@ -0,0 +1,373 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Loader for BAGEL-7B-MoT (ByteDance-Seed/BAGEL-7B-MoT) model.
+
+BAGEL uses a Qwen2-based LLM with MoT (Mixture of Transformers) extensions.
+Since transformers doesn't natively support the 'bagel' model_type, we construct
+the model manually by:
+ 1. Building a standard Qwen2ForCausalLM from the llm_config
+ 2. Adding MoT generation-path modules (mlp_moe_gen, *_moe_gen projections)
+ 3. Loading all weights from safetensors
+ 4. Wrapping in BagelForQuantization for auto_round compatibility
+"""
+
+import glob
+import json
+import os
+
+import torch
+import torch.nn as nn
+from transformers import AutoTokenizer, PretrainedConfig
+from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM, Qwen2MLP, Qwen2RMSNorm
+
+from auto_round.logger import logger
+
+
+class BagelConfig(PretrainedConfig):
+ """Configuration for the BAGEL model wrapper."""
+
+ model_type = "bagel"
+
+ def __init__(self, **kwargs):
+ super().__init__(**kwargs)
+
+
+def _add_mot_extensions(language_model, llm_config):
+ """Add MoT (Mixture of Transformers) generation-path modules to a Qwen2 model.
+
+ Each transformer layer gets additional modules for the generation path:
+ - Attention: q_proj_moe_gen, k_proj_moe_gen, v_proj_moe_gen, o_proj_moe_gen
+ - Attention norms: q_norm_moe_gen, k_norm_moe_gen (when qk_norm is used)
+ - MLP: mlp_moe_gen (full MLP duplicate)
+ - LayerNorms: input_layernorm_moe_gen, post_attention_layernorm_moe_gen
+ """
+ hidden_size = llm_config.hidden_size
+ num_heads = llm_config.num_attention_heads
+ num_kv_heads = getattr(llm_config, "num_key_value_heads", num_heads)
+ head_dim = hidden_size // num_heads
+ rms_norm_eps = llm_config.rms_norm_eps
+ use_qk_norm = getattr(llm_config, "qk_norm", False)
+
+ for layer in language_model.model.layers:
+ attn = layer.self_attn
+
+ # QK norms for the understanding path (not present in standard Qwen2Attention)
+ if use_qk_norm and not hasattr(attn, "q_norm"):
+ attn.q_norm = Qwen2RMSNorm(head_dim, eps=rms_norm_eps)
+ attn.k_norm = Qwen2RMSNorm(head_dim, eps=rms_norm_eps)
+
+ # MoT attention projections for generation path
+ attn.q_proj_moe_gen = nn.Linear(hidden_size, num_heads * head_dim, bias=True)
+ attn.k_proj_moe_gen = nn.Linear(hidden_size, num_kv_heads * head_dim, bias=True)
+ attn.v_proj_moe_gen = nn.Linear(hidden_size, num_kv_heads * head_dim, bias=True)
+ attn.o_proj_moe_gen = nn.Linear(num_heads * head_dim, hidden_size, bias=False)
+
+ # MoT QK norms for generation path
+ if use_qk_norm:
+ attn.q_norm_moe_gen = Qwen2RMSNorm(head_dim, eps=rms_norm_eps)
+ attn.k_norm_moe_gen = Qwen2RMSNorm(head_dim, eps=rms_norm_eps)
+ else:
+ attn.q_norm_moe_gen = nn.Identity()
+ attn.k_norm_moe_gen = nn.Identity()
+
+ # MoT MLP for generation path (duplicate of understanding MLP)
+ layer.mlp_moe_gen = Qwen2MLP(llm_config)
+
+ # MoT LayerNorms for generation path
+ layer.input_layernorm_moe_gen = Qwen2RMSNorm(hidden_size, eps=rms_norm_eps)
+ layer.post_attention_layernorm_moe_gen = Qwen2RMSNorm(hidden_size, eps=rms_norm_eps)
+
+
+def _build_module_from_weights(weight_dict):
+ """Build an nn.Module hierarchy from a flat weight dictionary.
+
+ This creates a generic module tree that holds parameters but doesn't
+ define forward operations. Used for non-text modules (VAE, ViT, etc.)
+ that need to be saved but not executed during quantization.
+ """
+ root = nn.Module()
+
+ # Group weights by first path component
+ children = {}
+ for name, tensor in weight_dict.items():
+ parts = name.split(".", 1)
+ if len(parts) == 1:
+ root.register_parameter(parts[0], nn.Parameter(tensor, requires_grad=False))
+ else:
+ child_name = parts[0]
+ if child_name not in children:
+ children[child_name] = {}
+ children[child_name][parts[1]] = tensor
+
+ for child_name, child_weights in children.items():
+ child_module = _build_module_from_weights(child_weights)
+ root.add_module(child_name, child_module)
+
+ return root
+
+
+def _load_safetensors_weights(model_path):
+ """Load all weights from safetensors files in the model directory.
+
+ BAGEL stores all weights across ae.safetensors (VAE) and ema.safetensors
+ (LLM + other modules), referenced by model.safetensors.index.json.
+ """
+ from safetensors.torch import load_file
+
+ all_weights = {}
+
+ index_path = os.path.join(model_path, "model.safetensors.index.json")
+ if os.path.exists(index_path):
+ with open(index_path, "r", encoding="utf-8") as f:
+ index = json.load(f)
+
+ weight_map = index.get("weight_map", {})
+
+ # Determine which shard files contain non-VAE weights
+ # VAE weights: decoder.*, encoder.* (in ae.safetensors)
+ lm_shard_files = set()
+ vae_only_files = set()
+ for weight_name, shard_file in weight_map.items():
+ if weight_name.startswith(("decoder.", "encoder.")):
+ vae_only_files.add(shard_file)
+ else:
+ lm_shard_files.add(shard_file)
+
+ # Load all shard files that contain non-VAE weights
+ loaded_files = set()
+ for shard_file in lm_shard_files:
+ if shard_file in loaded_files:
+ continue
+ sf_path = os.path.join(model_path, shard_file)
+ if os.path.exists(sf_path):
+ weights = load_file(sf_path, device="cpu")
+ # Only keep non-VAE weights from this file
+ for name, tensor in weights.items():
+ if not name.startswith(("decoder.", "encoder.")):
+ all_weights[name] = tensor
+ loaded_files.add(shard_file)
+ else:
+ # Fallback: load all safetensors files except ae.safetensors
+ for sf_file in sorted(glob.glob(os.path.join(model_path, "*.safetensors"))):
+ basename = os.path.basename(sf_file)
+ if basename == "ae.safetensors":
+ continue
+ weights = load_file(sf_file, device="cpu")
+ for name, tensor in weights.items():
+ if not name.startswith(("decoder.", "encoder.")):
+ all_weights[name] = tensor
+
+ return all_weights
+
+
+class BagelForQuantization(nn.Module):
+ """Wrapper for BAGEL model that's compatible with auto_round quantization.
+
+ Contains the language_model (Qwen2+MoT) as the primary quantization target,
+ plus non-text modules (connector, vit, etc.) stored as generic parameter holders.
+
+ The forward() delegates to language_model for text-only calibration.
+ """
+
+ def __init__(self, config, language_model, source_model_path=None):
+ super().__init__()
+ self.config = config
+ self.language_model = language_model
+ self._source_model_path = source_model_path
+
+ def forward(self, input_ids=None, attention_mask=None, labels=None, **kwargs):
+ """Forward pass delegating to the language_model for text-only calibration."""
+ return self.language_model(
+ input_ids=input_ids,
+ attention_mask=attention_mask,
+ labels=labels,
+ use_cache=kwargs.get("use_cache", False),
+ )
+
+ @property
+ def dtype(self):
+ return next(self.parameters()).dtype
+
+ @property
+ def device(self):
+ return next(self.parameters()).device
+
+ def get_input_embeddings(self):
+ return self.language_model.get_input_embeddings()
+
+ def save_pretrained(self, output_dir, **kwargs):
+ """Save the model in a format compatible with vllm-omni's BagelPipeline.
+
+ Saves:
+ - config.json: Original BAGEL config with quantization_config added
+ - model weights: All parameters as safetensors
+
+ Note: Auxiliary files (llm_config.json, vit_config.json,
+ preprocessor_config.json) and VAE tensors (encoder/decoder) are handled
+ by auto_round's _copy_extra_model_files and copy_missing_tensors_from_source.
+ """
+ from safetensors.torch import save_file
+
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Save config.json with the quantization_config
+ config_dict = self.config.to_dict()
+ # Remove internal PretrainedConfig fields
+ for key in list(config_dict.keys()):
+ if key.startswith("_"):
+ del config_dict[key]
+ config_dict["architectures"] = ["BagelForConditionalGeneration"]
+ config_dict["model_type"] = "bagel"
+ config_path = os.path.join(output_dir, "config.json")
+ with open(config_path, "w", encoding="utf-8") as f:
+ json.dump(config_dict, f, indent=2, ensure_ascii=False)
+
+ # Save all model parameters and buffers as safetensors
+ state_dict = {}
+ for name, param in self.named_parameters():
+ state_dict[name] = param.data.contiguous()
+ for name, buffer in self.named_buffers():
+ state_dict[name] = buffer.data.contiguous()
+
+ # Remap weight names to match original BAGEL checkpoint format
+ # The BagelPipeline expects top-level names like:
+ # language_model.model.layers.0.self_attn.q_proj.weight
+ # connector.fc1.weight
+ # vit_model.vision_model.embeddings...
+ # encoder.*, decoder.* (VAE, but those are in ae.safetensors)
+ save_file(state_dict, os.path.join(output_dir, "model.safetensors"))
+
+
+def load_bagel_model(model_path, torch_dtype="auto", device_map=None):
+ """Load a BAGEL model for quantization.
+
+ Args:
+ model_path: Path to the BAGEL model directory.
+ torch_dtype: Data type for model weights.
+ device_map: Device map for model placement.
+
+ Returns:
+ Tuple of (model, tokenizer).
+ """
+ # Load configs
+ config_path = os.path.join(model_path, "config.json")
+ with open(config_path, "r", encoding="utf-8") as f:
+ bagel_config_dict = json.load(f)
+
+ llm_config_dict = bagel_config_dict.get("llm_config", {})
+
+ # Check for separate llm_config.json
+ llm_config_path = os.path.join(model_path, "llm_config.json")
+ if os.path.exists(llm_config_path):
+ with open(llm_config_path, "r", encoding="utf-8") as f:
+ llm_config_dict = json.load(f)
+
+ from transformers import Qwen2Config
+
+ llm_config = Qwen2Config(**llm_config_dict)
+ # BAGEL always uses qk_norm
+ llm_config.qk_norm = True
+
+ # Determine torch_dtype
+ if torch_dtype == "auto":
+ model_dtype_str = bagel_config_dict.get("torch_dtype", "bfloat16")
+ if model_dtype_str == "bfloat16":
+ resolved_dtype = torch.bfloat16
+ elif model_dtype_str == "float16":
+ resolved_dtype = torch.float16
+ else:
+ resolved_dtype = torch.float32
+ elif isinstance(torch_dtype, str):
+ if torch_dtype == "bfloat16":
+ resolved_dtype = torch.bfloat16
+ elif torch_dtype == "float16":
+ resolved_dtype = torch.float16
+ elif torch_dtype == "float32":
+ resolved_dtype = torch.float32
+ else:
+ resolved_dtype = torch.float32
+ else:
+ resolved_dtype = torch_dtype
+
+ logger.info("Building Qwen2ForCausalLM with MoT extensions for BAGEL...")
+
+ # Create the language model (Qwen2 + MoT extensions)
+ language_model = Qwen2ForCausalLM(llm_config)
+ _add_mot_extensions(language_model, llm_config)
+
+ # Load all weights
+ logger.info(f"Loading weights from {model_path}...")
+ all_weights = _load_safetensors_weights(model_path)
+
+ # Separate language_model weights from other component weights
+ lm_weights = {}
+ other_weights = {}
+ for name, tensor in all_weights.items():
+ if name.startswith("language_model."):
+ lm_name = name[len("language_model.") :]
+ lm_weights[lm_name] = tensor
+ else:
+ other_weights[name] = tensor
+
+ # Load language_model weights
+ missing, unexpected = language_model.load_state_dict(lm_weights, strict=False)
+ if missing:
+ logger.warning(f"Missing keys in language_model: {len(missing)} keys")
+ for k in missing[:10]:
+ logger.warning(f" Missing: {k}")
+ if unexpected:
+ logger.warning(f"Unexpected keys in language_model: {len(unexpected)} keys")
+ for k in unexpected[:10]:
+ logger.warning(f" Unexpected: {k}")
+
+ # Build the BAGEL config
+ bagel_config = BagelConfig(
+ **{k: v for k, v in bagel_config_dict.items() if k not in ("llm_config", "architectures")}
+ )
+ bagel_config.llm_config = llm_config.to_dict()
+ bagel_config.architectures = ["BagelForConditionalGeneration"]
+
+ # Create the wrapper model
+ model = BagelForQuantization(bagel_config, language_model, source_model_path=model_path)
+
+ # Add non-text modules as parameter holders
+ # These won't be quantized but will be saved with the model
+ if other_weights:
+ non_text_module = _build_module_from_weights(other_weights)
+ for child_name, child_module in non_text_module.named_children():
+ if not hasattr(model, child_name):
+ model.add_module(child_name, child_module)
+ # Also add direct parameters
+ for param_name, param in non_text_module.named_parameters(recurse=False):
+ if not hasattr(model, param_name):
+ model.register_parameter(param_name, param)
+
+ # Convert to target dtype
+ model = model.to(resolved_dtype)
+ model.eval()
+
+ # Set name_or_path for auto_round compatibility
+ model.name_or_path = model_path
+ model.config._name_or_path = model_path
+
+ # Load tokenizer
+ tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
+
+ logger.info(
+ f"BAGEL model loaded: {sum(p.numel() for p in model.parameters()) / 1e9:.2f}B parameters, "
+ f"language_model has {llm_config.num_hidden_layers} layers"
+ )
+
+ return model, tokenizer
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 004a619c5..db7f2294f 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -336,6 +336,26 @@ def llm_load_model(
load_kwargs["quantization_config"] = Mxfp4Config(dequantized=True)
logger.info("Detected MXFP4 quantized model, using Mxfp4Config(dequantized=True) for loading.")
+ # BAGEL requires a custom loader (Qwen2 + not extensions, not in transformers)
+ _config_path = (
+ os.path.join(pretrained_model_name_or_path, "config.json")
+ if os.path.isdir(pretrained_model_name_or_path)
+ else None
+ )
+ if _config_path and os.path.exists(_config_path):
+ with open(_config_path) as _f:
+ _mt = json.load(_f).get("model_type")
+ if _mt == "bagel":
+ from auto_round.utils.bagel_loader import load_bagel_model
+
+ model, tokenizer = load_bagel_model(
+ pretrained_model_name_or_path,
+ torch_dtype=torch_dtype,
+ )
+ model = _to_model_dtype(model, model_dtype)
+ model._autoround_to_quant_block_names = "language_model.model.layers"
+ return model, tokenizer
+
is_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))
tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)
@@ -543,6 +563,15 @@ def mllm_load_model(
torch_dtype=torch_dtype,
device_map="auto" if use_auto_mapping else None,
)
+ elif "bagel" == model_type:
+ from auto_round.utils.bagel_loader import load_bagel_model
+
+ model, tokenizer = load_bagel_model(
+ pretrained_model_name_or_path,
+ torch_dtype=torch_dtype,
+ )
+ processor = None
+ image_processor = None
else:
architectures = config["architectures"][0]
if architectures == "LlavaLlamaForCausalLM":
@@ -760,6 +789,9 @@ def is_pure_text_model(model):
_is_mllm_model_cache: dict = {}
+# Model types that have multimodal components but should use LLM compressor
+# (text-only calibration, non-text modules excluded from quantization).
+_LLM_ONLY_MODEL_TYPES = {"bagel"}
def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = None):
@@ -771,6 +803,19 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = No
if model_path in _is_mllm_model_cache:
return _is_mllm_model_cache[model_path]
+ # Check model_type exclusion: some models have multimodal components
+ # but should be quantized as LLM (e.g., BAGEL MoT).
+ _model_type = None
+ if isinstance(model_or_path, torch.nn.Module) and hasattr(model_or_path, "config"):
+ _model_type = getattr(model_or_path.config, "model_type", None)
+ elif isinstance(model_path, str) and os.path.isdir(model_path):
+ _cfg_path = os.path.join(model_path, "config.json")
+ if os.path.exists(_cfg_path):
+ with open(_cfg_path) as _f:
+ _model_type = json.load(_f).get("model_type")
+ if _model_type in _LLM_ONLY_MODEL_TYPES:
+ return False
+
# For dummy model, model_path could be "".
# Only try to download if the path looks like a HF repo id (not a local filesystem path).
# Skip download for absolute paths or relative paths that contain current/parent dir markers.
@@ -1720,6 +1765,9 @@ def _get_reference_amax_from_experts(moe_module: torch.nn.Module, attr_name: str
# the quantized output directory so that from_pretrained() works out of the box.
_EXTRA_MODEL_FILES = {
"spk_dict.pt", # Qwen2.5-Omni speaker dictionary for audio output
+ "llm_config.json", # BAGEL sub-model config
+ "vit_config.json", # BAGEL vision transformer config
+ "preprocessor_config.json", # BAGEL image preprocessor config
}
diff --git a/test/test_cpu/models/test_bagel.py b/test/test_cpu/models/test_bagel.py
new file mode 100644
index 000000000..946851a8d
--- /dev/null
+++ b/test/test_cpu/models/test_bagel.py
@@ -0,0 +1,512 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit tests for BAGEL model support."""
+
+import json
+import os
+import shutil
+import tempfile
+
+import pytest
+import torch
+import torch.nn as nn
+from safetensors.torch import save_file
+
+# Absolute path to the repository root (repo_root/test/test_cpu/models/ -> repo_root)
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from auto_round.special_model_handler import (
+ _get_bagel_multimodal_block,
+ get_bagel_ignore_layers,
+ register_ignore_layers,
+)
+from auto_round.utils.model import (
+ _EXTRA_MODEL_FILES,
+ _LLM_ONLY_MODEL_TYPES,
+ get_block_names,
+ is_mllm_model,
+ llm_load_model,
+ mllm_load_model,
+)
+
+# ================= Fake BAGEL model for testing =================
+
+
+class FakeBagelConfig:
+ """Minimal fake config that mimics a BAGEL config.json."""
+
+ def __init__(self):
+ self.model_type = "bagel"
+ self.architectures = ["BagelForConditionalGeneration"]
+ self.hidden_size = 256
+ self.num_hidden_layers = 2
+ self.num_attention_heads = 4
+ self.num_key_value_heads = 2
+ self.intermediate_size = 512
+ self.rms_norm_eps = 1e-6
+ selfqk_norm = True
+
+ def to_dict(self):
+ return {
+ "model_type": self.model_type,
+ "architectures": self.architectures,
+ "hidden_size": self.hidden_size,
+ "num_hidden_layers": self.num_hidden_layers,
+ "num_attention_heads": self.num_attention_heads,
+ "num_key_value_heads": self.num_key_value_heads,
+ "intermediate_size": self.intermediate_size,
+ "rms_norm_eps": self.rms_norm_eps,
+ }
+
+
+def _make_fake_bagel_dir(num_layers=2, include_tokenizer=False):
+ """Create a temporary directory with fake BAGEL-style safetensors, config, and optionally a tokenizer."""
+ from transformers import AutoTokenizer
+
+ tmpdir = tempfile.mkdtemp()
+
+ # llm_config.json (Qwen2Config-like)
+ llm_config = {
+ "hidden_size": 256,
+ "intermediate_size": 512,
+ "num_hidden_layers": num_layers,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 2,
+ "rms_norm_eps": 1e-6,
+ "qk_norm": True,
+ "vocab_size": 1000,
+ }
+ with open(os.path.join(tmpdir, "llm_config.json"), "w") as f:
+ json.dump(llm_config, f)
+
+ # vit_config.json
+ vit_config = {"hidden_size": 128, "num_hidden_layers": 1}
+ with open(os.path.join(tmpdir, "vit_config.json"), "w") as f:
+ json.dump(vit_config, f)
+
+ # preprocessor_config.json
+ with open(os.path.join(tmpdir, "preprocessor_config.json"), "w") as f:
+ json.dump({}, f)
+
+ # Root config.json (BAGEL model_type)
+ config = {
+ "model_type": "bagel",
+ "architectures": ["BagelForConditionalGeneration"],
+ "llm_config": llm_config,
+ "torch_dtype": "bfloat16",
+ }
+ with open(os.path.join(tmpdir, "config.json"), "w") as f:
+ json.dump(config, f)
+
+ # Safetensors with language_model.* keys + non-language_model keys
+ state_dict = {}
+ # LLM params (keys use the BAGEL "language_model." prefix so bagel_loader
+ # strips it before calling language_model.load_state_dict)
+ for i in range(num_layers):
+ layer_prefix = f"language_model.model.layers.{i}."
+ state_dict[f"{layer_prefix}self_attn.q_proj.weight"] = torch.randn(256, 256)
+ state_dict[f"{layer_prefix}self_attn.q_proj.bias"] = torch.randn(256)
+ state_dict[f"{layer_prefix}self_attn.k_proj.weight"] = torch.randn(128, 256)
+ state_dict[f"{layer_prefix}self_attn.k_proj.bias"] = torch.randn(128)
+ state_dict[f"{layer_prefix}self_attn.v_proj.weight"] = torch.randn(128, 256)
+ state_dict[f"{layer_prefix}self_attn.v_proj.bias"] = torch.randn(128)
+ state_dict[f"{layer_prefix}self_attn.o_proj.weight"] = torch.randn(256, 256)
+ state_dict[f"{layer_prefix}mlp.gate_proj.weight"] = torch.randn(512, 256)
+ state_dict[f"{layer_prefix}mlp.up_proj.weight"] = torch.randn(512, 256)
+ state_dict[f"{layer_prefix}mlp.down_proj.weight"] = torch.randn(256, 512)
+ state_dict[f"{layer_prefix}input_layernorm.weight"] = torch.randn(256)
+ state_dict[f"{layer_prefix}post_attention_layernorm.weight"] = torch.randn(256)
+ # MOT generation-path params
+ state_dict[f"{layer_prefix}self_attn.q_proj_moe_gen.weight"] = torch.randn(256, 256)
+ state_dict[f"{layer_prefix}self_attn.q_proj_moe_gen.bias"] = torch.randn(256)
+ state_dict[f"{layer_prefix}self_attn.k_proj_moe_gen.weight"] = torch.randn(128, 256)
+ state_dict[f"{layer_prefix}self_attn.k_proj_moe_gen.bias"] = torch.randn(128)
+ state_dict[f"{layer_prefix}self_attn.v_proj_moe_gen.weight"] = torch.randn(128, 256)
+ state_dict[f"{layer_prefix}self_attn.v_proj_moe_gen.bias"] = torch.randn(128)
+ state_dict[f"{layer_prefix}self_attn.o_proj_moe_gen.weight"] = torch.randn(256, 256)
+ state_dict[f"{layer_prefix}mlp_moe_gen.gate_proj.weight"] = torch.randn(512, 256)
+ state_dict[f"{layer_prefix}mlp_moe_gen.up_proj.weight"] = torch.randn(512, 256)
+ state_dict[f"{layer_prefix}mlp_moe_gen.down_proj.weight"] = torch.randn(256, 512)
+ state_dict[f"{layer_prefix}input_layernorm_moe_gen.weight"] = torch.randn(256)
+ state_dict[f"{layer_prefix}post_attention_layernorm_moe_gen.weight"] = torch.randn(256)
+ # QK norms
+ state_dict[f"{layer_prefix}self_attn.q_norm.weight"] = torch.randn(64)
+ state_dict[f"{layer_prefix}self_attn.k_norm.weight"] = torch.randn(64)
+ state_dict[f"{layer_prefix}self_attn.q_norm_moe_gen.weight"] = torch.randn(64)
+ state_dict[f"{layer_prefix}self_attn.k_norm_moe_gen.weight"] = torch.randn(64)
+ state_dict["language_model.model.embed_tokens.weight"] = torch.randn(1000, 256)
+ state_dict["language_model.lm_head.weight"] = torch.randn(1000, 256)
+ state_dict["language_model.model.norm.weight"] = torch.randn(256)
+
+ # Non-language_model params (e.g., ViT / VAE)
+ state_dict["vit_model.vision_model.embeddings.patch_embedding.weight"] = torch.randn(128, 3, 16, 16)
+ state_dict["vit_model.vision_model.post_layernorm.weight"] = torch.randn(128)
+ state_dict["connector.fc1.weight"] = torch.randn(256, 128)
+ state_dict["encoder.conv_in.weight"] = torch.randn(128, 4, 3, 3)
+
+ save_file(state_dict, os.path.join(tmpdir, "model.safetensors"))
+
+ if include_tokenizer:
+ # Create a minimal tokenizer (Qwen vocab)
+ from transformers import AutoTokenizer
+
+ # Try to use an existing tiny tokenizer as base
+ try:
+ base_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B", trust_remote_code=False)
+ except Exception:
+ base_tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m", trust_remote_code=False)
+ base_tokenizer.save_pretrained(tmpdir)
+
+ return tmpdir
+
+
+# ================= Test: special_model_handler =================
+
+
+class TestBagelSpecialModelHandler:
+ """Tests for BAGEL-specific parts of special_model_handler.py."""
+
+ def test_bagel_multimodal_block_returns_language_model_layers(self):
+ """_get_bagel_multimodal_block should return block names for language_model layers."""
+
+ # BAGEL's language_model is a Qwen2ForCausalLM (which has .model containing layers).
+ # We create a minimal mock to avoid torch.nn.Module attribute resolution issues.
+ class MockLayers(list):
+ pass
+
+ class MockLanguageModel:
+ def __init__(self):
+ self.model = type("MockModel", (), {"layers": MockLayers([None, None])})()
+
+ class MockBagelModel:
+ def __init__(self):
+ self.language_model = MockLanguageModel()
+
+ model = MockBagelModel()
+ blocks = _get_bagel_multimodal_block(model)
+ assert len(blocks) == 1
+ assert len(blocks[0]) == 2
+ assert blocks[0][0] == "language_model.model.layers.0"
+ assert blocks[0][1] == "language_model.model.layers.1"
+
+ def test_get_bagel_ignore_layers_structure(self):
+ """get_bagel_ignore_layers should return generation-path modules."""
+
+ # Use minimal mock matching BAGEL's attribute structure
+ class MockLanguageModel:
+ def __init__(self):
+ self.model = type("MockModel", (), {"layers": [None, None, None]})()
+
+ class MockBagelModel:
+ def __init__(self):
+ self.language_model = MockLanguageModel()
+
+ model = MockBagelModel()
+ ignore = get_bagel_ignore_layers(model)
+ assert "moe_gen" in ignore
+ assert "self_attn.q_proj" in ignore
+ assert "self_attn.k_proj" in ignore
+ assert "self_attn.v_proj" in ignore
+ assert "self_attn.o_proj" in ignore
+
+ def test_bagel_registered_in_special_multimodal_block(self):
+ """BAGEL should be in SPECIAL_MULTIMODAL_BLOCK."""
+ from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK
+
+ assert "bagel" in SPECIAL_MULTIMODAL_BLOCK
+ handler = SPECIAL_MULTIMODAL_BLOCK["bagel"]
+ assert callable(handler)
+
+ def test_bagel_in_support_only_text_models(self):
+ """BAGEL should be in SUPPORT_ONLY_TEXT_MODELS."""
+ from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS
+
+ assert "bagel" in SUPPORT_ONLY_TEXT_MODELS
+
+
+# ================= Test: is_mllm_model =================
+
+
+class TestBagelIsMllm:
+ """Tests for BAGEL handling in is_mllm_model (utils/model.py)."""
+
+ def test_is_mllm_returns_false_for_bagel_model_type(self):
+ """is_mllm_model should return False for BAGEL (LLM-only quantization)."""
+ # BAGEL has multimodal components (ViT, VAE) but should be quantized as LLM
+ assert "bagel" in _LLM_ONLY_MODEL_TYPES
+ # Test via path
+ fake_dir = _make_fake_bagel_dir(num_layers=2)
+ try:
+ result = is_mllm_model(fake_dir)
+ assert result is False, "BAGEL should not be detected as MLLM"
+ finally:
+ shutil.rmtree(fake_dir)
+
+ def test_is_mllm_returns_false_for_bagel_model_object(self):
+ """is_mllm_model should return False for a BAGEL model object."""
+ from transformers import Qwen2Config
+
+ config = Qwen2Config(
+ hidden_size=256,
+ num_hidden_layers=2,
+ num_attention_heads=4,
+ num_key_value_heads=2,
+ intermediate_size=512,
+ )
+ from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM
+
+ model = Qwen2ForCausalLM(config)
+
+ class FakeBagelModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.lm_head = model.lm_head
+ self.name_or_path = "/fake/bagel/path"
+ config = Qwen2Config()
+ config.model_type = "bagel"
+ self.config = config
+
+ fake_model = FakeBagelModel()
+ result = is_mllm_model(fake_model)
+ assert result is False, "BAGEL model object should not be detected as MLLM"
+
+
+# ================= Test: _EXTRA_MODEL_FILES =================
+
+
+class TestBagelExtraModelFiles:
+ """Tests for BAGEL files in _EXTRA_MODEL_FILES."""
+
+ def test_extra_model_files_contain_bagel_configs(self):
+ """llm_config.json, vit_config.json, preprocessor_config.json should be in _EXTRA_MODEL_FILES."""
+ expected = {"llm_config.json", "vit_config.json", "preprocessor_config.json"}
+ assert expected.issubset(_EXTRA_MODEL_FILES), (
+ f"BAGEL extra files {expected} should be in _EXTRA_MODEL_FILES, " f"got {_EXTRA_MODEL_FILES}"
+ )
+
+
+# ================= Test: BagelConfig =================
+
+
+class TestBagelConfig:
+ """Tests for BagelConfig in bagel_loader.py."""
+
+ def test_bagel_config_model_type(self):
+ from auto_round.utils.bagel_loader import BagelConfig
+
+ config = BagelConfig()
+ assert config.model_type == "bagel"
+
+ def test_bagel_config_accepts_kwargs(self):
+ from auto_round.utils.bagel_loader import BagelConfig
+
+ config = BagelConfig(hidden_size=512, num_hidden_layers=4)
+ assert config.hidden_size == 512
+ assert config.num_hidden_layers == 4
+
+
+# ================= Test: BagelForQuantization save_pretrained =================
+
+
+class TestBagelForQuantization:
+ """Tests for BagelForQuantization wrapper (bagel_loader.py)."""
+
+ @pytest.fixture
+ def fake_bagel_dir(self):
+ d = _make_fake_bagel_dir(num_layers=2, include_tokenizer=True)
+ yield d
+ shutil.rmtree(d, ignore_errors=True)
+
+ @pytest.fixture
+ def bagel_model_tokenizer(self, fake_bagel_dir):
+ from auto_round.utils.bagel_loader import load_bagel_model
+
+ model, tokenizer = load_bagel_model(fake_bagel_dir, torch_dtype=torch.bfloat16)
+ return model, tokenizer
+
+ def test_load_bagel_model_returns_model_and_tokenizer(self, bagel_model_tokenizer):
+ """load_bagel_model should return (model, tokenizer)."""
+ model, tokenizer = bagel_model_tokenizer
+ assert model is not None
+ assert tokenizer is not None
+
+ def test_load_bagel_model_has_language_model(self, bagel_model_tokenizer):
+ """Loaded model should have language_model attribute."""
+ model, _ = bagel_model_tokenizer
+ assert hasattr(model, "language_model")
+
+ def test_load_bagel_model_has_bagel_config(self, bagel_model_tokenizer):
+ """Loaded model config should have bagel model_type."""
+ model, _ = bagel_model_tokenizer
+ assert model.config.model_type == "bagel"
+
+ def test_load_bagel_model_sets_name_or_path(self, bagel_model_tokenizer, fake_bagel_dir):
+ """Loaded model should have name_or_path set."""
+ model, _ = bagel_model_tokenizer
+ assert model.name_or_path == fake_bagel_dir
+
+ def test_load_bagel_model_sets_autoround_block_hint(self, fake_bagel_dir):
+ """llm_load_model should set _autoround_to_quant_block_names for BAGEL."""
+ model, _ = llm_load_model(fake_bagel_dir)
+ assert hasattr(model, "_autoround_to_quant_block_names")
+ assert model._autoround_to_quant_block_names == "language_model.model.layers"
+ del model
+
+ def test_load_bagel_model_dtype_auto_bfloat16(self, fake_bagel_dir):
+ """torch_dtype='auto' should resolve bfloat16 from config."""
+ from auto_round.utils.bagel_loader import load_bagel_model
+
+ model, _ = load_bagel_model(fake_bagel_dir, torch_dtype="auto")
+ assert model is not None
+
+ def test_save_pretrained_includes_parameters(self, bagel_model_tokenizer, tmp_path):
+ """save_pretrained should include all named_parameters."""
+ model, _ = bagel_model_tokenizer
+ save_dir = str(tmp_path / "saved_bagel")
+
+ model.save_pretrained(save_dir)
+
+ assert os.path.exists(os.path.join(save_dir, "config.json"))
+ assert os.path.exists(os.path.join(save_dir, "model.safetensors"))
+
+ # Check config has bagel model_type
+ with open(os.path.join(save_dir, "config.json")) as f:
+ saved_config = json.load(f)
+ assert saved_config["model_type"] == "bagel"
+ assert "BagelForConditionalGeneration" in saved_config["architectures"]
+
+ # Check safetensors has parameters
+ from safetensors.torch import load_file
+
+ state = load_file(os.path.join(save_dir, "model.safetensors"))
+ # Should contain at least the LLM params we loaded
+ layer_keys = [k for k in state.keys() if "language_model.model.layers" in k]
+ assert len(layer_keys) > 0, "Safetensors should contain language_model layer params"
+
+ def test_save_pretrained_includes_buffers(self, bagel_model_tokenizer, tmp_path):
+ """save_pretrained should include named_buffers (not just parameters).
+
+ Regression test: previously only named_parameters was iterated.
+ """
+ model, _ = bagel_model_tokenizer
+
+ # Add a real buffer to the model
+ model.register_buffer("test_buffer", torch.randn(8, 8))
+ assert "test_buffer" in [n for n, _ in model.named_buffers()]
+
+ save_dir = str(tmp_path / "saved_bagel_buffers")
+ model.save_pretrained(save_dir)
+
+ from safetensors.torch import load_file
+
+ state = load_file(os.path.join(save_dir, "model.safetensors"))
+ # The buffer we added should be in the saved state_dict
+ assert "test_buffer" in state, "named_buffers should be included in save_pretrained state_dict"
+
+
+# ================= Test: compressor base - to_quant_block_names hint =================
+
+
+class TestBagelCompressorHint:
+ """Tests for _autoround_to_quant_block_names hint propagation in BaseCompressor."""
+
+ def test_compressor_picks_up_autoround_hint_when_none(self):
+ """Code path: to_quant_block_names=None should use model._autoround_to_quant_block_names."""
+ # Verify the logic directly from the source
+ to_quant_block_names = None
+ hint = "transformer.h"
+ expected = hint
+
+ result = to_quant_block_names
+ if result is None:
+ _hint = hint # simulating getattr(model, "_autoround_to_quant_block_names", None)
+ if _hint is not None:
+ result = _hint
+
+ assert result == expected
+
+ def test_compressor_uses_explicit_over_hint(self):
+ """Code path: explicit to_quant_block_names should override the hint."""
+ to_quant_block_names = "explicit.path"
+ hint = "transformer.h"
+
+ result = to_quant_block_names
+ if result is None:
+ _hint = hint
+ if _hint is not None:
+ result = _hint
+
+ assert result == "explicit.path"
+
+ def test_compressor_hint_parsed_in_base_compressor_source(self):
+ """Verify BaseCompressor reads _autoround_to_quant_block_names from model."""
+ source_path = os.path.join(REPO_ROOT, "auto_round/compressors/base.py")
+ with open(source_path) as f:
+ source = f.read()
+
+ assert "_autoround_to_quant_block_names" in source, "BaseCompressor should read _autoround_to_quant_block_names"
+
+
+# ================= Test: ValueError in AutoConfig.from_pretrained =================
+
+
+class TestConfigErrorHandling:
+ """Tests that ValueError is handled when loading config via AutoConfig (branch diff)."""
+
+ def test_autoconfig_valueerror_caught_in_base_compressor(self):
+ """BaseCompressor.__init__: ValueError should be caught in AutoConfig.from_pretrained."""
+ source_path = os.path.join(REPO_ROOT, "auto_round/compressors/base.py")
+ with open(source_path) as f:
+ content = f.read()
+
+ # The branch diff adds ValueError to the except clause around line 294
+ # Verify the pattern appears (line numbers may shift)
+ assert (
+ "except (OSError, EnvironmentError, ValueError)" in content
+ ), "BaseCompressor should catch ValueError alongside OSError/EnvironmentError"
+
+ def test_autoconfig_valueerror_caught_in_model_context(self):
+ """ModelContext: ValueError should be caught in AutoConfig.from_pretrained."""
+ source_path = os.path.join(REPO_ROOT, "auto_round/context/model.py")
+ with open(source_path) as f:
+ content = f.read()
+
+ # The branch diff adds ValueError to the except clause around line 146
+ assert (
+ "except (OSError, EnvironmentError, ValueError)" in content
+ ), "ModelContext should catch ValueError alongside OSError/EnvironmentError"
+
+
+# ================= Test: mllm_load_model for bagel =================
+
+
+class TestBagelMllmLoadModel:
+ """Tests for mllm_load_model handling of bagel model_type."""
+
+ @pytest.fixture
+ def fake_bagel_dir(self):
+ d = _make_fake_bagel_dir(num_layers=2, include_tokenizer=True)
+ yield d
+ shutil.rmtree(d, ignore_errors=True)
+
+ def test_mllm_load_model_handles_bagel(self, fake_bagel_dir):
+ """mllm_load_model should load BAGEL via bagel_loader (not raise)."""
+ model, processor, tokenizer, image_processor = mllm_load_model(fake_bagel_dir)
+
+ assert model is not None
+ assert model.config.model_type == "bagel"
+ assert tokenizer is not None
+ # processor/image_processor should be None for BAGEL
+ assert processor is None
+ assert image_processor is None