PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support ByteDance-Seed/BAGEL-7B-MoT quantization in w4a16 format (#1633)
Liang Lv91679a5c30ea
diff --git a/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt b/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txtindex 41b7edde1..c829fc71b 100644--- a/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt+++ b/.azure-pipelines/scripts/codeScan/codespell/autoround_dict.txt@@ -1 +1,3 @@-endianess\ No newline at end of file+endianess+MOT+MoT\ No newline at end of filediff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex d70010dc1..65e2158b2 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -19,7 +19,7 @@ from dataclasses import asdict, dataclass, fieldsfrom typing import Any, Optional, Unionimport torch-from transformers import set_seed+from transformers import AutoConfig, set_seedfrom auto_round.algorithms.alg_config import AlgConfigfrom auto_round.algorithms.quantization import BaseQuantizers, QuantizationConfig@@ -127,6 +127,22 @@ class BaseCompressor(object):_scheme_resolved: bool = Falsescheme_generator = None+ @staticmethod+ def _preload_model_config(model: Union[torch.nn.Module, str], trust_remote_code: bool) -> Optional[AutoConfig]:+ if not isinstance(model, str):+ return None++ try:+ return AutoConfig.from_pretrained(model, trust_remote_code=trust_remote_code)+ except (OSError, EnvironmentError, ValueError) as e:+ logger.debug(+ "Failed to load config via AutoConfig.from_pretrained for %s: %s. "+ "Proceeding without config-based checks.",+ model,+ e,+ )+ return None+def __init__(self,config: Union[AlgConfig, list[AlgConfig]],@@ -273,6 +289,7 @@ class BaseCompressor(object):# allocation early in the heap, matching the OLD arch allocation order# and reducing C-heap fragmentation (which is amplified on HPU)._device = get_major_device(device_map if device_map is not None else 0)+ model_config = self._preload_model_config(model, trust_remote_code)self.model_context = ModelContext(model,@@ -280,6 +297,7 @@ class BaseCompressor(object):platform=platform,model_dtype=model_dtype,trust_remote_code=trust_remote_code,+ config=model_config,amp=amp,need_calib=self.need_calib,device=_device,@@ -739,6 +757,10 @@ class BaseCompressor(object):self.ignore_layers = cfg.ignore_layersself.quant_lm_head = cfg.quant_lm_headself.to_quant_block_names = cfg.to_quant_block_names+ if self.to_quant_block_names is None:+ self.to_quant_block_names = getattr(self.model_context.model, "_autoround_to_quant_block_names", None)+ if self.to_quant_block_names is not None:+ self.quantize_config.to_quant_block_names = self.to_quant_block_names# Resolve the scheme (pure config work: sets data_type / bits / sym /# scale_dtype etc. on both self and self.quantize_config).diff --git a/auto_round/context/model.py b/auto_round/context/model.pyindex 9bb46152c..a29db4073 100644--- a/auto_round/context/model.py+++ b/auto_round/context/model.py@@ -62,6 +62,7 @@ class ModelContext(BaseContext):platform="hf",model_dtype=None,trust_remote_code=True,+ config: Optional[AutoConfig] = None,amp=True,need_calib=True,device="cpu",@@ -92,6 +93,7 @@ class ModelContext(BaseContext):self.platform = platformself.model_dtype = model_dtypeself.trust_remote_code = trust_remote_code+ self.config = configself.amp = ampself.need_calib = need_calibself.quant_nontext_module = quant_nontext_module@@ -139,11 +141,12 @@ class ModelContext(BaseContext):self.model, platform=self.platform, device="cpu", model_dtype=self.model_dtype)elif isinstance(self.model, str):- config: Optional[AutoConfig] = None+ config = self.configtry:- config = AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)+ if config is None:+ config = AutoConfig.from_pretrained(self.model, trust_remote_code=self.trust_remote_code)self._import_custom_moe_replacements(config)- except (OSError, EnvironmentError) as e:+ except (OSError, EnvironmentError, ValueError) as e:logger.debug("Failed to load config via AutoConfig.from_pretrained for %s: %s. ""Proceeding without config-based checks.",diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.pyindex 264c77ab8..6f11b454c 100644--- a/auto_round/special_model_handler.py+++ b/auto_round/special_model_handler.py@@ -49,6 +49,7 @@ SUPPORT_ONLY_TEXT_MODELS = ["qwen3_omni_moe","gemma3","gemma4",+ "bagel",]NOT_SUPPORT_ONLY_TEXT_MODELS = ["mllama", "mistral3_2"]@@ -336,11 +337,35 @@ def _get_glm_image_multimodal_block(model, quant_vision=False):return block_names+def _get_bagel_multimodal_block(model, quant_vision=False):+ """Get block names for BAGEL MoT (Mixture of Transformers) model.++ BAGEL model structure:+ - language_model.model.layers: Qwen2-based LLM with MoT dual paths+ - vit_model: SigLIP vision encoder (not quantized by default)+ - connector: Vision-language MLP connector+ - encoder/decoder: VAE autoencoder+ - time_embedder, vae2llm, llm2vae: bridge modules++ By default, only the language_model layers are quantized.+ """+ block_names = []++ if hasattr(model, "language_model") and hasattr(model.language_model, "model"):+ if hasattr(model.language_model.model, "layers"):+ block_names.append(+ [f"language_model.model.layers.{i}" for i in range(len(model.language_model.model.layers))]+ )++ return block_names++SPECIAL_MULTIMODAL_BLOCK = {"deepseek_vl_v2": _get_deepseek_vl2_multimodal_block,"qwen2_5_omni": _get_qwen2_5_omni_multimodal_block,"qwen3_omni_moe": _get_qwen3_omni_moe_multimodal_block,"glm_image": _get_glm_image_multimodal_block,+ "bagel": _get_bagel_multimodal_block,}@@ -721,6 +746,46 @@ register_ignore_layers()+def get_bagel_ignore_layers(model) -> list[str]:+ """Keep BAGEL generation-path modules in FP16.++ BAGEL uses `*_moe_gen` modules for the image-generation path. Quantizing+ them causes quality to collapse during the iterative denoising loop.+ The shared attention projections are also highly sensitive, and preserving+ the top 4 transformer blocks in FP16 gave acceptable image quality in+ validation runs.+ """+ top_fp16_layers = 0++ ignore_layers = [+ "moe_gen",+ "self_attn.q_proj",+ "self_attn.k_proj",+ "self_attn.v_proj",+ "self_attn.o_proj",+ ]++ num_layers = 0+ if hasattr(model, "language_model") and hasattr(model.language_model, "model"):+ num_layers = len(getattr(model.language_model.model, "layers", []))++ if num_layers > 0:+ for layer_idx in range(max(0, num_layers - top_fp16_layers), num_layers):+ ignore_layers.append(f"language_model.model.layers.{layer_idx}")++ return ignore_layers+++register_ignore_layers(+ matchers=[+ ModelTypeMatcher(r"bagel", mode="full"),+ ],+ ignore_layers=[+ get_bagel_ignore_layers,+ ],+)++def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:layers = []for rule in _PRE_DEFINED_IGNORE_LAYERS:diff --git a/auto_round/utils/bagel_loader.py b/auto_round/utils/bagel_loader.pynew file mode 100644index 000000000..a4fb42599--- /dev/null+++ b/auto_round/utils/bagel_loader.py@@ -0,0 +1,373 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Loader for BAGEL-7B-MoT (ByteDance-Seed/BAGEL-7B-MoT) model.++BAGEL uses a Qwen2-based LLM with MoT (Mixture of Transformers) extensions.+Since transformers doesn't natively support the 'bagel' model_type, we construct+the model manually by:+ 1. Building a standard Qwen2ForCausalLM from the llm_config+ 2. Adding MoT generation-path modules (mlp_moe_gen, *_moe_gen projections)+ 3. Loading all weights from safetensors+ 4. Wrapping in BagelForQuantization for auto_round compatibility+"""++import glob+import json+import os++import torch+import torch.nn as nn+from transformers import AutoTokenizer, PretrainedConfig+from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM, Qwen2MLP, Qwen2RMSNorm++from auto_round.logger import logger+++class BagelConfig(PretrainedConfig):+ """Configuration for the BAGEL model wrapper."""++ model_type = "bagel"++ def __init__(self, **kwargs):+ super().__init__(**kwargs)+++def _add_mot_extensions(language_model, llm_config):+ """Add MoT (Mixture of Transformers) generation-path modules to a Qwen2 model.++ Each transformer layer gets additional modules for the generation path:+ - Attention: q_proj_moe_gen, k_proj_moe_gen, v_proj_moe_gen, o_proj_moe_gen+ - Attention norms: q_norm_moe_gen, k_norm_moe_gen (when qk_norm is used)+ - MLP: mlp_moe_gen (full MLP duplicate)+ - LayerNorms: input_layernorm_moe_gen, post_attention_layernorm_moe_gen+ """+ hidden_size = llm_config.hidden_size+ num_heads = llm_config.num_attention_heads+ num_kv_heads = getattr(llm_config, "num_key_value_heads", num_heads)+ head_dim = hidden_size // num_heads+ rms_norm_eps = llm_config.rms_norm_eps+ use_qk_norm = getattr(llm_config, "qk_norm", False)++ for layer in language_model.model.layers:+ attn = layer.self_attn++ # QK norms for the understanding path (not present in standard Qwen2Attention)+ if use_qk_norm and not hasattr(attn, "q_norm"):+ attn.q_norm = Qwen2RMSNorm(head_dim, eps=rms_norm_eps)+ attn.k_norm = Qwen2RMSNorm(head_dim, eps=rms_norm_eps)++ # MoT attention projections for generation path+ attn.q_proj_moe_gen = nn.Linear(hidden_size, num_heads * head_dim, bias=True)+ attn.k_proj_moe_gen = nn.Linear(hidden_size, num_kv_heads * head_dim, bias=True)+ attn.v_proj_moe_gen = nn.Linear(hidden_size, num_kv_heads * head_dim, bias=True)+ attn.o_proj_moe_gen = nn.Linear(num_heads * head_dim, hidden_size, bias=False)++ # MoT QK norms for generation path+ if use_qk_norm:+ attn.q_norm_moe_gen = Qwen2RMSNorm(head_dim, eps=rms_norm_eps)+ attn.k_norm_moe_gen = Qwen2RMSNorm(head_dim, eps=rms_norm_eps)+ else:+ attn.q_norm_moe_gen = nn.Identity()+ attn.k_norm_moe_gen = nn.Identity()++ # MoT MLP for generation path (duplicate of understanding MLP)+ layer.mlp_moe_gen = Qwen2MLP(llm_config)++ # MoT LayerNorms for generation path+ layer.input_layernorm_moe_gen = Qwen2RMSNorm(hidden_size, eps=rms_norm_eps)+ layer.post_attention_layernorm_moe_gen = Qwen2RMSNorm(hidden_size, eps=rms_norm_eps)+++def _build_module_from_weights(weight_dict):+ """Build an nn.Module hierarchy from a flat weight dictionary.++ This creates a generic module tree that holds parameters but doesn't+ define forward operations. Used for non-text modules (VAE, ViT, etc.)+ that need to be saved but not executed during quantization.+ """+ root = nn.Module()++ # Group weights by first path component+ children = {}+ for name, tensor in weight_dict.items():+ parts = name.split(".", 1)+ if len(parts) == 1:+ root.register_parameter(parts[0], nn.Parameter(tensor, requires_grad=False))+ else:+ child_name = parts[0]+ if child_name not in children:+ children[child_name] = {}+ children[child_name][parts[1]] = tensor++ for child_name, child_weights in children.items():+ child_module = _build_module_from_weights(child_weights)+ root.add_module(child_name, child_module)++ return root+++def _load_safetensors_weights(model_path):+ """Load all weights from safetensors files in the model directory.++ BAGEL stores all weights across ae.safetensors (VAE) and ema.safetensors+ (LLM + other modules), referenced by model.safetensors.index.json.+ """+ from safetensors.torch import load_file++ all_weights = {}++ index_path = os.path.join(model_path, "model.safetensors.index.json")+ if os.path.exists(index_path):+ with open(index_path, "r", encoding="utf-8") as f:+ index = json.load(f)++ weight_map = index.get("weight_map", {})++ # Determine which shard files contain non-VAE weights+ # VAE weights: decoder.*, encoder.* (in ae.safetensors)+ lm_shard_files = set()+ vae_only_files = set()+ for weight_name, shard_file in weight_map.items():+ if weight_name.startswith(("decoder.", "encoder.")):+ vae_only_files.add(shard_file)+ else:+ lm_shard_files.add(shard_file)++ # Load all shard files that contain non-VAE weights+ loaded_files = set()+ for shard_file in lm_shard_files:+ if shard_file in loaded_files:+ continue+ sf_path = os.path.join(model_path, shard_file)+ if os.path.exists(sf_path):+ weights = load_file(sf_path, device="cpu")+ # Only keep non-VAE weights from this file+ for name, tensor in weights.items():+ if not name.startswith(("decoder.", "encoder.")):+ all_weights[name] = tensor+ loaded_files.add(shard_file)+ else:+ # Fallback: load all safetensors files except ae.safetensors+ for sf_file in sorted(glob.glob(os.path.join(model_path, "*.safetensors"))):+ basename = os.path.basename(sf_file)+ if basename == "ae.safetensors":+ continue+ weights = load_file(sf_file, device="cpu")+ for name, tensor in weights.items():+ if not name.startswith(("decoder.", "encoder.")):+ all_weights[name] = tensor++ return all_weights+++class BagelForQuantization(nn.Module):+ """Wrapper for BAGEL model that's compatible with auto_round quantization.++ Contains the language_model (Qwen2+MoT) as the primary quantization target,+ plus non-text modules (connector, vit, etc.) stored as generic parameter holders.++ The forward() delegates to language_model for text-only calibration.+ """++ def __init__(self, config, language_model, source_model_path=None):+ super().__init__()+ self.config = config+ self.language_model = language_model+ self._source_model_path = source_model_path++ def forward(self, input_ids=None, attention_mask=None, labels=None, **kwargs):+ """Forward pass delegating to the language_model for text-only calibration."""+ return self.language_model(+ input_ids=input_ids,+ attention_mask=attention_mask,+ labels=labels,+ use_cache=kwargs.get("use_cache", False),+ )++ @property+ def dtype(self):+ return next(self.parameters()).dtype++ @property+ def device(self):+ return next(self.parameters()).device++ def get_input_embeddings(self):+ return self.language_model.get_input_embeddings()++ def save_pretrained(self, output_dir, **kwargs):+ """Save the model in a format compatible with vllm-omni's BagelPipeline.++ Saves:+ - config.json: Original BAGEL config with quantization_config added+ - model weights: All parameters as safetensors++ Note: Auxiliary files (llm_config.json, vit_config.json,+ preprocessor_config.json) and VAE tensors (encoder/decoder) are handled+ by auto_round's _copy_extra_model_files and copy_missing_tensors_from_source.+ """+ from safetensors.torch import save_file++ os.makedirs(output_dir, exist_ok=True)++ # Save config.json with the quantization_config+ config_dict = self.config.to_dict()+ # Remove internal PretrainedConfig fields+ for key in list(config_dict.keys()):+ if key.startswith("_"):+ del config_dict[key]+ config_dict["architectures"] = ["BagelForConditionalGeneration"]+ config_dict["model_type"] = "bagel"+ config_path = os.path.join(output_dir, "config.json")+ with open(config_path, "w", encoding="utf-8") as f:+ json.dump(config_dict, f, indent=2, ensure_ascii=False)++ # Save all model parameters and buffers as safetensors+ state_dict = {}+ for name, param in self.named_parameters():+ state_dict[name] = param.data.contiguous()+ for name, buffer in self.named_buffers():+ state_dict[name] = buffer.data.contiguous()++ # Remap weight names to match original BAGEL checkpoint format+ # The BagelPipeline expects top-level names like:+ # language_model.model.layers.0.self_attn.q_proj.weight+ # connector.fc1.weight+ # vit_model.vision_model.embeddings...+ # encoder.*, decoder.* (VAE, but those are in ae.safetensors)+ save_file(state_dict, os.path.join(output_dir, "model.safetensors"))+++def load_bagel_model(model_path, torch_dtype="auto", device_map=None):+ """Load a BAGEL model for quantization.++ Args:+ model_path: Path to the BAGEL model directory.+ torch_dtype: Data type for model weights.+ device_map: Device map for model placement.++ Returns:+ Tuple of (model, tokenizer).+ """+ # Load configs+ config_path = os.path.join(model_path, "config.json")+ with open(config_path, "r", encoding="utf-8") as f:+ bagel_config_dict = json.load(f)++ llm_config_dict = bagel_config_dict.get("llm_config", {})++ # Check for separate llm_config.json+ llm_config_path = os.path.join(model_path, "llm_config.json")+ if os.path.exists(llm_config_path):+ with open(llm_config_path, "r", encoding="utf-8") as f:+ llm_config_dict = json.load(f)++ from transformers import Qwen2Config++ llm_config = Qwen2Config(**llm_config_dict)+ # BAGEL always uses qk_norm+ llm_config.qk_norm = True++ # Determine torch_dtype+ if torch_dtype == "auto":+ model_dtype_str = bagel_config_dict.get("torch_dtype", "bfloat16")+ if model_dtype_str == "bfloat16":+ resolved_dtype = torch.bfloat16+ elif model_dtype_str == "float16":+ resolved_dtype = torch.float16+ else:+ resolved_dtype = torch.float32+ elif isinstance(torch_dtype, str):+ if torch_dtype == "bfloat16":+ resolved_dtype = torch.bfloat16+ elif torch_dtype == "float16":+ resolved_dtype = torch.float16+ elif torch_dtype == "float32":+ resolved_dtype = torch.float32+ else:+ resolved_dtype = torch.float32+ else:+ resolved_dtype = torch_dtype++ logger.info("Building Qwen2ForCausalLM with MoT extensions for BAGEL...")++ # Create the language model (Qwen2 + MoT extensions)+ language_model = Qwen2ForCausalLM(llm_config)+ _add_mot_extensions(language_model, llm_config)++ # Load all weights+ logger.info(f"Loading weights from {model_path}...")+ all_weights = _load_safetensors_weights(model_path)++ # Separate language_model weights from other component weights+ lm_weights = {}+ other_weights = {}+ for name, tensor in all_weights.items():+ if name.startswith("language_model."):+ lm_name = name[len("language_model.") :]+ lm_weights[lm_name] = tensor+ else:+ other_weights[name] = tensor++ # Load language_model weights+ missing, unexpected = language_model.load_state_dict(lm_weights, strict=False)+ if missing:+ logger.warning(f"Missing keys in language_model: {len(missing)} keys")+ for k in missing[:10]:+ logger.warning(f" Missing: {k}")+ if unexpected:+ logger.warning(f"Unexpected keys in language_model: {len(unexpected)} keys")+ for k in unexpected[:10]:+ logger.warning(f" Unexpected: {k}")++ # Build the BAGEL config+ bagel_config = BagelConfig(+ **{k: v for k, v in bagel_config_dict.items() if k not in ("llm_config", "architectures")}+ )+ bagel_config.llm_config = llm_config.to_dict()+ bagel_config.architectures = ["BagelForConditionalGeneration"]++ # Create the wrapper model+ model = BagelForQuantization(bagel_config, language_model, source_model_path=model_path)++ # Add non-text modules as parameter holders+ # These won't be quantized but will be saved with the model+ if other_weights:+ non_text_module = _build_module_from_weights(other_weights)+ for child_name, child_module in non_text_module.named_children():+ if not hasattr(model, child_name):+ model.add_module(child_name, child_module)+ # Also add direct parameters+ for param_name, param in non_text_module.named_parameters(recurse=False):+ if not hasattr(model, param_name):+ model.register_parameter(param_name, param)++ # Convert to target dtype+ model = model.to(resolved_dtype)+ model.eval()++ # Set name_or_path for auto_round compatibility+ model.name_or_path = model_path+ model.config._name_or_path = model_path++ # Load tokenizer+ tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)++ logger.info(+ f"BAGEL model loaded: {sum(p.numel() for p in model.parameters()) / 1e9:.2f}B parameters, "+ f"language_model has {llm_config.num_hidden_layers} layers"+ )++ return model, tokenizerdiff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 004a619c5..db7f2294f 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -336,6 +336,26 @@ def llm_load_model(load_kwargs["quantization_config"] = Mxfp4Config(dequantized=True)logger.info("Detected MXFP4 quantized model, using Mxfp4Config(dequantized=True) for loading.")+ # BAGEL requires a custom loader (Qwen2 + not extensions, not in transformers)+ _config_path = (+ os.path.join(pretrained_model_name_or_path, "config.json")+ if os.path.isdir(pretrained_model_name_or_path)+ else None+ )+ if _config_path and os.path.exists(_config_path):+ with open(_config_path) as _f:+ _mt = json.load(_f).get("model_type")+ if _mt == "bagel":+ from auto_round.utils.bagel_loader import load_bagel_model++ model, tokenizer = load_bagel_model(+ pretrained_model_name_or_path,+ torch_dtype=torch_dtype,+ )+ model = _to_model_dtype(model, model_dtype)+ model._autoround_to_quant_block_names = "language_model.model.layers"+ return model, tokenizer+is_glm = bool(re.search("chatglm", pretrained_model_name_or_path.lower()))tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, trust_remote_code=trust_remote_code)@@ -543,6 +563,15 @@ def mllm_load_model(torch_dtype=torch_dtype,device_map="auto" if use_auto_mapping else None,)+ elif "bagel" == model_type:+ from auto_round.utils.bagel_loader import load_bagel_model++ model, tokenizer = load_bagel_model(+ pretrained_model_name_or_path,+ torch_dtype=torch_dtype,+ )+ processor = None+ image_processor = Noneelse:architectures = config["architectures"][0]if architectures == "LlavaLlamaForCausalLM":@@ -760,6 +789,9 @@ def is_pure_text_model(model):_is_mllm_model_cache: dict = {}+# Model types that have multimodal components but should use LLM compressor+# (text-only calibration, non-text modules excluded from quantization).+_LLM_ONLY_MODEL_TYPES = {"bagel"}def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = None):@@ -771,6 +803,19 @@ def is_mllm_model(model_or_path: Union[str, torch.nn.Module], platform: str = Noif model_path in _is_mllm_model_cache:return _is_mllm_model_cache[model_path]+ # Check model_type exclusion: some models have multimodal components+ # but should be quantized as LLM (e.g., BAGEL MoT).+ _model_type = None+ if isinstance(model_or_path, torch.nn.Module) and hasattr(model_or_path, "config"):+ _model_type = getattr(model_or_path.config, "model_type", None)+ elif isinstance(model_path, str) and os.path.isdir(model_path):+ _cfg_path = os.path.join(model_path, "config.json")+ if os.path.exists(_cfg_path):+ with open(_cfg_path) as _f:+ _model_type = json.load(_f).get("model_type")+ if _model_type in _LLM_ONLY_MODEL_TYPES:+ return False+# For dummy model, model_path could be "".# Only try to download if the path looks like a HF repo id (not a local filesystem path).# Skip download for absolute paths or relative paths that contain current/parent dir markers.@@ -1720,6 +1765,9 @@ def _get_reference_amax_from_experts(moe_module: torch.nn.Module, attr_name: str# the quantized output directory so that from_pretrained() works out of the box._EXTRA_MODEL_FILES = {"spk_dict.pt", # Qwen2.5-Omni speaker dictionary for audio output+ "llm_config.json", # BAGEL sub-model config+ "vit_config.json", # BAGEL vision transformer config+ "preprocessor_config.json", # BAGEL image preprocessor config}diff --git a/test/test_cpu/models/test_bagel.py b/test/test_cpu/models/test_bagel.pynew file mode 100644index 000000000..946851a8d--- /dev/null+++ b/test/test_cpu/models/test_bagel.py@@ -0,0 +1,512 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+"""Unit tests for BAGEL model support."""++import json+import os+import shutil+import tempfile++import pytest+import torch+import torch.nn as nn+from safetensors.torch import save_file++# Absolute path to the repository root (repo_root/test/test_cpu/models/ -> repo_root)+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))++from auto_round.special_model_handler import (+ _get_bagel_multimodal_block,+ get_bagel_ignore_layers,+ register_ignore_layers,+)+from auto_round.utils.model import (+ _EXTRA_MODEL_FILES,+ _LLM_ONLY_MODEL_TYPES,+ get_block_names,+ is_mllm_model,+ llm_load_model,+ mllm_load_model,+)++# ================= Fake BAGEL model for testing =================+++class FakeBagelConfig:+ """Minimal fake config that mimics a BAGEL config.json."""++ def __init__(self):+ self.model_type = "bagel"+ self.architectures = ["BagelForConditionalGeneration"]+ self.hidden_size = 256+ self.num_hidden_layers = 2+ self.num_attention_heads = 4+ self.num_key_value_heads = 2+ self.intermediate_size = 512+ self.rms_norm_eps = 1e-6+ selfqk_norm = True++ def to_dict(self):+ return {+ "model_type": self.model_type,+ "architectures": self.architectures,+ "hidden_size": self.hidden_size,+ "num_hidden_layers": self.num_hidden_layers,+ "num_attention_heads": self.num_attention_heads,+ "num_key_value_heads": self.num_key_value_heads,+ "intermediate_size": self.intermediate_size,+ "rms_norm_eps": self.rms_norm_eps,+ }+++def _make_fake_bagel_dir(num_layers=2, include_tokenizer=False):+ """Create a temporary directory with fake BAGEL-style safetensors, config, and optionally a tokenizer."""+ from transformers import AutoTokenizer++ tmpdir = tempfile.mkdtemp()++ # llm_config.json (Qwen2Config-like)+ llm_config = {+ "hidden_size": 256,+ "intermediate_size": 512,+ "num_hidden_layers": num_layers,+ "num_attention_heads": 4,+ "num_key_value_heads": 2,+ "rms_norm_eps": 1e-6,+ "qk_norm": True,+ "vocab_size": 1000,+ }+ with open(os.path.join(tmpdir, "llm_config.json"), "w") as f:+ json.dump(llm_config, f)++ # vit_config.json+ vit_config = {"hidden_size": 128, "num_hidden_layers": 1}+ with open(os.path.join(tmpdir, "vit_config.json"), "w") as f:+ json.dump(vit_config, f)++ # preprocessor_config.json+ with open(os.path.join(tmpdir, "preprocessor_config.json"), "w") as f:+ json.dump({}, f)++ # Root config.json (BAGEL model_type)+ config = {+ "model_type": "bagel",+ "architectures": ["BagelForConditionalGeneration"],+ "llm_config": llm_config,+ "torch_dtype": "bfloat16",+ }+ with open(os.path.join(tmpdir, "config.json"), "w") as f:+ json.dump(config, f)++ # Safetensors with language_model.* keys + non-language_model keys+ state_dict = {}+ # LLM params (keys use the BAGEL "language_model." prefix so bagel_loader+ # strips it before calling language_model.load_state_dict)+ for i in range(num_layers):+ layer_prefix = f"language_model.model.layers.{i}."+ state_dict[f"{layer_prefix}self_attn.q_proj.weight"] = torch.randn(256, 256)+ state_dict[f"{layer_prefix}self_attn.q_proj.bias"] = torch.randn(256)+ state_dict[f"{layer_prefix}self_attn.k_proj.weight"] = torch.randn(128, 256)+ state_dict[f"{layer_prefix}self_attn.k_proj.bias"] = torch.randn(128)+ state_dict[f"{layer_prefix}self_attn.v_proj.weight"] = torch.randn(128, 256)+ state_dict[f"{layer_prefix}self_attn.v_proj.bias"] = torch.randn(128)+ state_dict[f"{layer_prefix}self_attn.o_proj.weight"] = torch.randn(256, 256)+ state_dict[f"{layer_prefix}mlp.gate_proj.weight"] = torch.randn(512, 256)+ state_dict[f"{layer_prefix}mlp.up_proj.weight"] = torch.randn(512, 256)+ state_dict[f"{layer_prefix}mlp.down_proj.weight"] = torch.randn(256, 512)+ state_dict[f"{layer_prefix}input_layernorm.weight"] = torch.randn(256)+ state_dict[f"{layer_prefix}post_attention_layernorm.weight"] = torch.randn(256)+ # MOT generation-path params+ state_dict[f"{layer_prefix}self_attn.q_proj_moe_gen.weight"] = torch.randn(256, 256)+ state_dict[f"{layer_prefix}self_attn.q_proj_moe_gen.bias"] = torch.randn(256)+ state_dict[f"{layer_prefix}self_attn.k_proj_moe_gen.weight"] = torch.randn(128, 256)+ state_dict[f"{layer_prefix}self_attn.k_proj_moe_gen.bias"] = torch.randn(128)+ state_dict[f"{layer_prefix}self_attn.v_proj_moe_gen.weight"] = torch.randn(128, 256)+ state_dict[f"{layer_prefix}self_attn.v_proj_moe_gen.bias"] = torch.randn(128)+ state_dict[f"{layer_prefix}self_attn.o_proj_moe_gen.weight"] = torch.randn(256, 256)+ state_dict[f"{layer_prefix}mlp_moe_gen.gate_proj.weight"] = torch.randn(512, 256)+ state_dict[f"{layer_prefix}mlp_moe_gen.up_proj.weight"] = torch.randn(512, 256)+ state_dict[f"{layer_prefix}mlp_moe_gen.down_proj.weight"] = torch.randn(256, 512)+ state_dict[f"{layer_prefix}input_layernorm_moe_gen.weight"] = torch.randn(256)+ state_dict[f"{layer_prefix}post_attention_layernorm_moe_gen.weight"] = torch.randn(256)+ # QK norms+ state_dict[f"{layer_prefix}self_attn.q_norm.weight"] = torch.randn(64)+ state_dict[f"{layer_prefix}self_attn.k_norm.weight"] = torch.randn(64)+ state_dict[f"{layer_prefix}self_attn.q_norm_moe_gen.weight"] = torch.randn(64)+ state_dict[f"{layer_prefix}self_attn.k_norm_moe_gen.weight"] = torch.randn(64)+ state_dict["language_model.model.embed_tokens.weight"] = torch.randn(1000, 256)+ state_dict["language_model.lm_head.weight"] = torch.randn(1000, 256)+ state_dict["language_model.model.norm.weight"] = torch.randn(256)++ # Non-language_model params (e.g., ViT / VAE)+ state_dict["vit_model.vision_model.embeddings.patch_embedding.weight"] = torch.randn(128, 3, 16, 16)+ state_dict["vit_model.vision_model.post_layernorm.weight"] = torch.randn(128)+ state_dict["connector.fc1.weight"] = torch.randn(256, 128)+ state_dict["encoder.conv_in.weight"] = torch.randn(128, 4, 3, 3)++ save_file(state_dict, os.path.join(tmpdir, "model.safetensors"))++ if include_tokenizer:+ # Create a minimal tokenizer (Qwen vocab)+ from transformers import AutoTokenizer++ # Try to use an existing tiny tokenizer as base+ try:+ base_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2-0.5B", trust_remote_code=False)+ except Exception:+ base_tokenizer = AutoTokenizer.from_pretrained("facebook/opt-125m", trust_remote_code=False)+ base_tokenizer.save_pretrained(tmpdir)++ return tmpdir+++# ================= Test: special_model_handler =================+++class TestBagelSpecialModelHandler:+ """Tests for BAGEL-specific parts of special_model_handler.py."""++ def test_bagel_multimodal_block_returns_language_model_layers(self):+ """_get_bagel_multimodal_block should return block names for language_model layers."""++ # BAGEL's language_model is a Qwen2ForCausalLM (which has .model containing layers).+ # We create a minimal mock to avoid torch.nn.Module attribute resolution issues.+ class MockLayers(list):+ pass++ class MockLanguageModel:+ def __init__(self):+ self.model = type("MockModel", (), {"layers": MockLayers([None, None])})()++ class MockBagelModel:+ def __init__(self):+ self.language_model = MockLanguageModel()++ model = MockBagelModel()+ blocks = _get_bagel_multimodal_block(model)+ assert len(blocks) == 1+ assert len(blocks[0]) == 2+ assert blocks[0][0] == "language_model.model.layers.0"+ assert blocks[0][1] == "language_model.model.layers.1"++ def test_get_bagel_ignore_layers_structure(self):+ """get_bagel_ignore_layers should return generation-path modules."""++ # Use minimal mock matching BAGEL's attribute structure+ class MockLanguageModel:+ def __init__(self):+ self.model = type("MockModel", (), {"layers": [None, None, None]})()++ class MockBagelModel:+ def __init__(self):+ self.language_model = MockLanguageModel()++ model = MockBagelModel()+ ignore = get_bagel_ignore_layers(model)+ assert "moe_gen" in ignore+ assert "self_attn.q_proj" in ignore+ assert "self_attn.k_proj" in ignore+ assert "self_attn.v_proj" in ignore+ assert "self_attn.o_proj" in ignore++ def test_bagel_registered_in_special_multimodal_block(self):+ """BAGEL should be in SPECIAL_MULTIMODAL_BLOCK."""+ from auto_round.special_model_handler import SPECIAL_MULTIMODAL_BLOCK++ assert "bagel" in SPECIAL_MULTIMODAL_BLOCK+ handler = SPECIAL_MULTIMODAL_BLOCK["bagel"]+ assert callable(handler)++ def test_bagel_in_support_only_text_models(self):+ """BAGEL should be in SUPPORT_ONLY_TEXT_MODELS."""+ from auto_round.special_model_handler import SUPPORT_ONLY_TEXT_MODELS++ assert "bagel" in SUPPORT_ONLY_TEXT_MODELS+++# ================= Test: is_mllm_model =================+++class TestBagelIsMllm:+ """Tests for BAGEL handling in is_mllm_model (utils/model.py)."""++ def test_is_mllm_returns_false_for_bagel_model_type(self):+ """is_mllm_model should return False for BAGEL (LLM-only quantization)."""+ # BAGEL has multimodal components (ViT, VAE) but should be quantized as LLM+ assert "bagel" in _LLM_ONLY_MODEL_TYPES+ # Test via path+ fake_dir = _make_fake_bagel_dir(num_layers=2)+ try:+ result = is_mllm_model(fake_dir)+ assert result is False, "BAGEL should not be detected as MLLM"+ finally:+ shutil.rmtree(fake_dir)++ def test_is_mllm_returns_false_for_bagel_model_object(self):+ """is_mllm_model should return False for a BAGEL model object."""+ from transformers import Qwen2Config++ config = Qwen2Config(+ hidden_size=256,+ num_hidden_layers=2,+ num_attention_heads=4,+ num_key_value_heads=2,+ intermediate_size=512,+ )+ from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM++ model = Qwen2ForCausalLM(config)++ class FakeBagelModel(nn.Module):+ def __init__(self):+ super().__init__()+ self.lm_head = model.lm_head+ self.name_or_path = "/fake/bagel/path"+ config = Qwen2Config()+ config.model_type = "bagel"+ self.config = config++ fake_model = FakeBagelModel()+ result = is_mllm_model(fake_model)+ assert result is False, "BAGEL model object should not be detected as MLLM"+++# ================= Test: _EXTRA_MODEL_FILES =================+++class TestBagelExtraModelFiles:+ """Tests for BAGEL files in _EXTRA_MODEL_FILES."""++ def test_extra_model_files_contain_bagel_configs(self):+ """llm_config.json, vit_config.json, preprocessor_config.json should be in _EXTRA_MODEL_FILES."""+ expected = {"llm_config.json", "vit_config.json", "preprocessor_config.json"}+ assert expected.issubset(_EXTRA_MODEL_FILES), (+ f"BAGEL extra files {expected} should be in _EXTRA_MODEL_FILES, " f"got {_EXTRA_MODEL_FILES}"+ )+++# ================= Test: BagelConfig =================+++class TestBagelConfig:+ """Tests for BagelConfig in bagel_loader.py."""++ def test_bagel_config_model_type(self):+ from auto_round.utils.bagel_loader import BagelConfig++ config = BagelConfig()+ assert config.model_type == "bagel"++ def test_bagel_config_accepts_kwargs(self):+ from auto_round.utils.bagel_loader import BagelConfig++ config = BagelConfig(hidden_size=512, num_hidden_layers=4)+ assert config.hidden_size == 512+ assert config.num_hidden_layers == 4+++# ================= Test: BagelForQuantization save_pretrained =================+++class TestBagelForQuantization:+ """Tests for BagelForQuantization wrapper (bagel_loader.py)."""++ @pytest.fixture+ def fake_bagel_dir(self):+ d = _make_fake_bagel_dir(num_layers=2, include_tokenizer=True)+ yield d+ shutil.rmtree(d, ignore_errors=True)++ @pytest.fixture+ def bagel_model_tokenizer(self, fake_bagel_dir):+ from auto_round.utils.bagel_loader import load_bagel_model++ model, tokenizer = load_bagel_model(fake_bagel_dir, torch_dtype=torch.bfloat16)+ return model, tokenizer++ def test_load_bagel_model_returns_model_and_tokenizer(self, bagel_model_tokenizer):+ """load_bagel_model should return (model, tokenizer)."""+ model, tokenizer = bagel_model_tokenizer+ assert model is not None+ assert tokenizer is not None++ def test_load_bagel_model_has_language_model(self, bagel_model_tokenizer):+ """Loaded model should have language_model attribute."""+ model, _ = bagel_model_tokenizer+ assert hasattr(model, "language_model")++ def test_load_bagel_model_has_bagel_config(self, bagel_model_tokenizer):+ """Loaded model config should have bagel model_type."""+ model, _ = bagel_model_tokenizer+ assert model.config.model_type == "bagel"++ def test_load_bagel_model_sets_name_or_path(self, bagel_model_tokenizer, fake_bagel_dir):+ """Loaded model should have name_or_path set."""+ model, _ = bagel_model_tokenizer+ assert model.name_or_path == fake_bagel_dir++ def test_load_bagel_model_sets_autoround_block_hint(self, fake_bagel_dir):+ """llm_load_model should set _autoround_to_quant_block_names for BAGEL."""+ model, _ = llm_load_model(fake_bagel_dir)+ assert hasattr(model, "_autoround_to_quant_block_names")+ assert model._autoround_to_quant_block_names == "language_model.model.layers"+ del model++ def test_load_bagel_model_dtype_auto_bfloat16(self, fake_bagel_dir):+ """torch_dtype='auto' should resolve bfloat16 from config."""+ from auto_round.utils.bagel_loader import load_bagel_model++ model, _ = load_bagel_model(fake_bagel_dir, torch_dtype="auto")+ assert model is not None++ def test_save_pretrained_includes_parameters(self, bagel_model_tokenizer, tmp_path):+ """save_pretrained should include all named_parameters."""+ model, _ = bagel_model_tokenizer+ save_dir = str(tmp_path / "saved_bagel")++ model.save_pretrained(save_dir)++ assert os.path.exists(os.path.join(save_dir, "config.json"))+ assert os.path.exists(os.path.join(save_dir, "model.safetensors"))++ # Check config has bagel model_type+ with open(os.path.join(save_dir, "config.json")) as f:+ saved_config = json.load(f)+ assert saved_config["model_type"] == "bagel"+ assert "BagelForConditionalGeneration" in saved_config["architectures"]++ # Check safetensors has parameters+ from safetensors.torch import load_file++ state = load_file(os.path.join(save_dir, "model.safetensors"))+ # Should contain at least the LLM params we loaded+ layer_keys = [k for k in state.keys() if "language_model.model.layers" in k]+ assert len(layer_keys) > 0, "Safetensors should contain language_model layer params"++ def test_save_pretrained_includes_buffers(self, bagel_model_tokenizer, tmp_path):+ """save_pretrained should include named_buffers (not just parameters).++ Regression test: previously only named_parameters was iterated.+ """+ model, _ = bagel_model_tokenizer++ # Add a real buffer to the model+ model.register_buffer("test_buffer", torch.randn(8, 8))+ assert "test_buffer" in [n for n, _ in model.named_buffers()]++ save_dir = str(tmp_path / "saved_bagel_buffers")+ model.save_pretrained(save_dir)++ from safetensors.torch import load_file++ state = load_file(os.path.join(save_dir, "model.safetensors"))+ # The buffer we added should be in the saved state_dict+ assert "test_buffer" in state, "named_buffers should be included in save_pretrained state_dict"+++# ================= Test: compressor base - to_quant_block_names hint =================+++class TestBagelCompressorHint:+ """Tests for _autoround_to_quant_block_names hint propagation in BaseCompressor."""++ def test_compressor_picks_up_autoround_hint_when_none(self):+ """Code path: to_quant_block_names=None should use model._autoround_to_quant_block_names."""+ # Verify the logic directly from the source+ to_quant_block_names = None+ hint = "transformer.h"+ expected = hint++ result = to_quant_block_names+ if result is None:+ _hint = hint # simulating getattr(model, "_autoround_to_quant_block_names", None)+ if _hint is not None:+ result = _hint++ assert result == expected++ def test_compressor_uses_explicit_over_hint(self):+ """Code path: explicit to_quant_block_names should override the hint."""+ to_quant_block_names = "explicit.path"+ hint = "transformer.h"++ result = to_quant_block_names+ if result is None:+ _hint = hint+ if _hint is not None:+ result = _hint++ assert result == "explicit.path"++ def test_compressor_hint_parsed_in_base_compressor_source(self):+ """Verify BaseCompressor reads _autoround_to_quant_block_names from model."""+ source_path = os.path.join(REPO_ROOT, "auto_round/compressors/base.py")+ with open(source_path) as f:+ source = f.read()++ assert "_autoround_to_quant_block_names" in source, "BaseCompressor should read _autoround_to_quant_block_names"+++# ================= Test: ValueError in AutoConfig.from_pretrained =================+++class TestConfigErrorHandling:+ """Tests that ValueError is handled when loading config via AutoConfig (branch diff)."""++ def test_autoconfig_valueerror_caught_in_base_compressor(self):+ """BaseCompressor.__init__: ValueError should be caught in AutoConfig.from_pretrained."""+ source_path = os.path.join(REPO_ROOT, "auto_round/compressors/base.py")+ with open(source_path) as f:+ content = f.read()++ # The branch diff adds ValueError to the except clause around line 294+ # Verify the pattern appears (line numbers may shift)+ assert (+ "except (OSError, EnvironmentError, ValueError)" in content+ ), "BaseCompressor should catch ValueError alongside OSError/EnvironmentError"++ def test_autoconfig_valueerror_caught_in_model_context(self):+ """ModelContext: ValueError should be caught in AutoConfig.from_pretrained."""+ source_path = os.path.join(REPO_ROOT, "auto_round/context/model.py")+ with open(source_path) as f:+ content = f.read()++ # The branch diff adds ValueError to the except clause around line 146+ assert (+ "except (OSError, EnvironmentError, ValueError)" in content+ ), "ModelContext should catch ValueError alongside OSError/EnvironmentError"+++# ================= Test: mllm_load_model for bagel =================+++class TestBagelMllmLoadModel:+ """Tests for mllm_load_model handling of bagel model_type."""++ @pytest.fixture+ def fake_bagel_dir(self):+ d = _make_fake_bagel_dir(num_layers=2, include_tokenizer=True)+ yield d+ shutil.rmtree(d, ignore_errors=True)++ def test_mllm_load_model_handles_bagel(self, fake_bagel_dir):+ """mllm_load_model should load BAGEL via bagel_loader (not raise)."""+ model, processor, tokenizer, image_processor = mllm_load_model(fake_bagel_dir)++ assert model is not None+ assert model.config.model_type == "bagel"+ assert tokenizer is not None+ # processor/image_processor should be None for BAGEL+ assert processor is None+ assert image_processor is None