Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support qwen3_5 moe (#1476)

Wenhua Cheng1b37c316987d
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index de58ec36a..8bdae36b2 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -27,9 +27,10 @@ import accelerate
import torch
from accelerate.big_modeling import dispatch_model, infer_auto_device_map
from accelerate.utils import get_balanced_memory, get_max_memory
+from packaging import version
from torch import autocast
from tqdm import tqdm
-from transformers import set_seed
+from transformers import AutoConfig, set_seed
from auto_round import envs
from auto_round.auto_scheme.gen_auto_scheme import AutoScheme
@@ -92,6 +93,7 @@ from auto_round.utils import (
is_debug_mode,
is_hpex_available,
is_moe_model,
+ is_moe_model_via_config,
is_quantized_input_module,
llm_load_model,
memory_monitor,
@@ -272,23 +274,35 @@ class BaseCompressor(object):
self.quantized = False
self.is_model_patched = False
if isinstance(model, str):
+ config: Optional[AutoConfig] = None
try:
- # config = AutoConfig.from_pretrained(model)
- self.is_model_patched = apply_model_monkey_patches(model_name=model)
-
- # TODO excluded ori_params_moe
- # if (
- # not self.is_model_patched
- # and is_moe_model_via_config(config)
- # and version.parse(transformers.__version__) >= version.parse("5.0.0")
- # ):
- # logger.warning(
- # "This moe model is not optimized by AutoRound yet which may cause large ram usage, "
- # "please submit an issue to https://github.com/intel/auto-round/issues"
- # )
-
- except:
- pass
+ config = AutoConfig.from_pretrained(model, trust_remote_code=self.trust_remote_code)
+ except (OSError, EnvironmentError) as e:
+ logger.debug(
+ "Failed to load config via AutoConfig.from_pretrained for %s: %s. "
+ "Proceeding without config-based checks.",
+ model,
+ e,
+ )
+
+ self.is_model_patched = apply_model_monkey_patches(model_name=model)
+ import transformers
+
+ if (
+ not self.is_model_patched
+ and config is not None
+ and is_moe_model_via_config(config)
+ and version.parse(transformers.__version__) >= version.parse("5.0.0")
+ ):
+ from auto_round.modeling.fused_moe.replace_modules import BUILTIN_MODULES
+
+ model_type = getattr(config, "model_type", None)
+ if model_type is not None and model_type not in BUILTIN_MODULES:
+ logger.warning(
+ "This MoE model has not been optimized by AutoRound yet, which may result in high RAM usage, "
+ "Please consider submitting an issue to https://github.com/intel/auto-round/issues"
+ )
+
model, tokenizer = llm_load_model(
model,
platform=platform,
@@ -1341,7 +1355,9 @@ class BaseCompressor(object):
for block_name in block_names:
pbar.set_description(f"Quantizing {block_name}")
block = get_module(self.model, block_name)
+
materialize_model_(block)
+
for name, m in block.named_modules():
if hasattr(m, "global_name") and m.global_name in all_to_quantized_module_names:
self._quantize_layer_via_rtn(m.global_name, to_cpu=self.low_gpu_mem_usage)
@@ -2159,79 +2175,82 @@ class BaseCompressor(object):
all_inputs = self.cache_inter_data(block_names, nsamples, layer_names=[], last_cache_name=last_cache_name)
else:
try:
- if not self.model.device.type == "meta":
- if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
- self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)
- else:
- # Change this if new device is supported
- if str(self.model.device) == "cpu" and (not self.device.startswith("hpu")):
- # type(self.model._no_split_modules) changes from list to set when transformers > 5.0
- no_split_modules = list(getattr(self.model, "_no_split_modules", []))
- devices = parse_available_devices(self.device_map)
-
- max_memory = get_max_memory()
- new_max_memory = {}
- if "cpu" not in devices:
- devices.append("cpu")
- for device in devices:
- if ":" in device:
- device = int(device.split(":")[-1])
- elif device == "cpu":
- device = "cpu"
- elif isinstance(device, str):
- device = 0
- else:
- raise ValueError(f"Unsupported device {device} in device_map: {self.device_map}")
- if device not in max_memory:
- # Skip devices that are not reported by accelerate's max_memory.
- # This is expected when a device is unavailable or cannot provide memory info.
- continue
- # Use 90% of the reported max memory to leave headroom for activations,
- # temporary tensors, other processes, and allocator fragmentation, reducing
- # the chance of runtime OOM while still utilizing most available memory.
- new_max_memory[device] = max_memory[device] * 0.9
-
- # If non-CPU devices were requested but none survived, fall back to CPU caching
- # via the OOM handler below, avoiding unnecessary dispatch overhead.
- requested_non_cpu = any((d != "cpu") for d in devices)
- has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)
- if requested_non_cpu and not has_non_cpu_memory:
- raise torch.OutOfMemoryError(
- "No non-CPU device available in accelerate's reported memory. "
- "Falling back to CPU caching."
- )
+ if any(p.device.type == "meta" for p in self.model.parameters()):
+ materialize_model_(self.model)
- new_max_memory = get_balanced_memory(
- self.model,
- max_memory=new_max_memory,
- no_split_module_classes=no_split_modules,
+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:
+ self.model = dispatch_model(self.model, device_map=self.model.hf_device_map)
+ else:
+ # Change this if new device is supported
+ if str(self.model.device) == "cpu" and (not self.device.startswith("hpu")):
+ # type(self.model._no_split_modules) changes from list to set when transformers > 5.0
+ no_split_modules = list(getattr(self.model, "_no_split_modules", []))
+ devices = parse_available_devices(self.device_map)
+
+ max_memory = get_max_memory()
+ new_max_memory = {}
+ if "cpu" not in devices:
+ devices.append("cpu")
+ for device in devices:
+ if ":" in device:
+ device = int(device.split(":")[-1])
+ elif device == "cpu":
+ device = "cpu"
+ elif isinstance(device, str):
+ device = 0
+ else:
+ raise ValueError(f"Unsupported device {device} in device_map: {self.device_map}")
+ if device not in max_memory:
+ # Skip devices that are not reported by accelerate's max_memory.
+ # This is expected when a device is unavailable or cannot provide memory info.
+ continue
+ # Use 90% of the reported max memory to leave headroom for activations,
+ # temporary tensors, other processes, and allocator fragmentation, reducing
+ # the chance of runtime OOM while still utilizing most available memory.
+ new_max_memory[device] = max_memory[device] * 0.9
+
+ # If non-CPU devices were requested but none survived, fall back to CPU caching
+ # via the OOM handler below, avoiding unnecessary dispatch overhead.
+ requested_non_cpu = any((d != "cpu") for d in devices)
+ has_non_cpu_memory = any((k != "cpu") for k in new_max_memory)
+ if requested_non_cpu and not has_non_cpu_memory:
+ raise torch.OutOfMemoryError(
+ "No non-CPU device available in accelerate's reported memory. "
+ "Falling back to CPU caching."
)
- self.model.tie_weights()
- device_map = infer_auto_device_map(
- self.model, max_memory=new_max_memory, no_split_module_classes=no_split_modules
+
+ new_max_memory = get_balanced_memory(
+ self.model,
+ max_memory=new_max_memory,
+ no_split_module_classes=no_split_modules,
+ )
+ self.model.tie_weights()
+ device_map = infer_auto_device_map(
+ self.model, max_memory=new_max_memory, no_split_module_classes=no_split_modules
+ )
+ if len(devices) > 1 and "cpu" in device_map.values():
+ logger.warning(
+ "Some layers are offloaded to cpu, which may severely impact calibration speed."
+ " Please consider using more cards."
)
- if len(devices) > 1 and "cpu" in device_map.values():
+
+ try:
+
+ self.model = dispatch_model(self.model, device_map=device_map)
+ except ValueError as e:
+ if "offload_dir" in e.__str__():
logger.warning(
- "Some layers are offloaded to cpu, which may severely impact calibration speed."
- " Please consider using more cards."
+ f"Due to insufficient resources, disk is used to store the model."
+ f" `offload_dir={envs.AR_WORK_SPACE}`"
)
+ self.model = dispatch_model(
+ self.model, device_map=device_map, offload_dir=envs.AR_WORK_SPACE
+ )
+ else:
+ raise
+ else:
- try:
- materialize_model_(self.model)
- self.model = dispatch_model(self.model, device_map=device_map)
- except ValueError as e:
- if "offload_dir" in e.__str__():
- logger.warning(
- f"Due to insufficient resources, disk is used to store the model."
- f" `offload_dir={envs.AR_WORK_SPACE}`"
- )
- self.model = dispatch_model(
- self.model, device_map=device_map, offload_dir=envs.AR_WORK_SPACE
- )
- else:
- raise
- else:
- self.model = self.model.to(self.device)
+ self.model = self.model.to(self.device)
all_inputs = self.cache_inter_data(
block_names, nsamples, layer_names=layer_names, last_cache_name=last_cache_name
diff --git a/auto_round/modeling/fused_moe/qwen3_5_moe.py b/auto_round/modeling/fused_moe/qwen3_5_moe.py
new file mode 100644
index 000000000..b9fc560a7
--- /dev/null
+++ b/auto_round/modeling/fused_moe/qwen3_5_moe.py
@@ -0,0 +1,113 @@
+# # Copyright (C) 2026 Intel Corporation
+# # SPDX-License-Identifier: Apache-2.0
+
+import torch
+import torch.nn.functional as F
+from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeMLP
+from transformers.utils.versions import require_version
+
+from auto_round.modeling.fused_moe.replace_modules import ReplacementModuleBase
+from auto_round.utils import clear_memory, unsupported_meta_device
+
+require_version("transformers>=5.2.0")
+
+from auto_round.modeling.fused_moe.utils import _update_parameter
+
+
+class LinearQwen3_5MoeSparseMoeBlock(ReplacementModuleBase):
+ def __init__(self, original, config):
+ super().__init__(original)
+ self.gate = original.gate
+ text_config = config.get_text_config()
+ self.shared_expert = original.shared_expert
+ with torch.device("meta"):
+ self.experts = SequentialQwen3_5MoeExperts(text_config, original.experts)
+ self.shared_expert_gate = original.shared_expert_gate
+ self.num_experts = text_config.num_experts
+
+ @classmethod
+ def original_module_class(cls) -> str:
+ """Return the class name of the module this replaces."""
+ return "Qwen3_5MoeSparseMoeBlock"
+
+ def _materialize_weights(self) -> None:
+ original = self._get_original_module()
+ self.experts._materialize_weights(original.experts)
+ clear_memory()
+
+ def experts_forward(
+ self,
+ hidden_states: torch.Tensor,
+ top_k_index: torch.Tensor,
+ top_k_weights: torch.Tensor,
+ ) -> torch.Tensor:
+ final_hidden_states = torch.zeros_like(hidden_states)
+ with torch.no_grad():
+ expert_mask = torch.nn.functional.one_hot(top_k_index, num_classes=self.num_experts)
+ expert_mask = expert_mask.permute(2, 1, 0)
+ expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero()
+
+ for expert_idx in expert_hit:
+ expert_idx = expert_idx[0]
+ if expert_idx == self.num_experts:
+ continue
+ top_k_pos, token_idx = torch.where(expert_mask[expert_idx])
+ current_state = hidden_states[token_idx]
+ # gate, up = nn.functional.linear(current_state, self.gate_up_proj[expert_idx]).chunk(2, dim=-1)
+ # current_hidden_states = self.act_fn(gate) * up
+ # current_hidden_states = nn.functional.linear(current_hidden_states, self.down_proj[expert_idx])
+ current_hidden_states = self.experts[expert_idx](current_state)
+ current_hidden_states = current_hidden_states * top_k_weights[token_idx, top_k_pos, None]
+ final_hidden_states.index_add_(0, token_idx, current_hidden_states.to(final_hidden_states.dtype))
+
+ return final_hidden_states
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ batch_size, sequence_length, hidden_dim = hidden_states.shape
+ hidden_states_reshaped = hidden_states.view(-1, hidden_dim)
+ shared_expert_output = self.shared_expert(hidden_states_reshaped)
+ _, routing_weights, selected_experts = self.gate(hidden_states_reshaped)
+ expert_output = self.experts_forward(hidden_states_reshaped, selected_experts, routing_weights)
+
+ shared_expert_output = F.sigmoid(self.shared_expert_gate(hidden_states_reshaped)) * shared_expert_output
+
+ expert_output += shared_expert_output
+ expert_output = expert_output.reshape(batch_size, sequence_length, hidden_dim)
+ return expert_output
+
+ @classmethod
+ def from_original(
+ cls,
+ original,
+ config,
+ **kwargs,
+ ):
+ """Create an instance from the original module."""
+ return cls(original, config)
+
+
+class SequentialQwen3_5MoeExperts(torch.nn.ModuleList):
+ def __init__(self, config, original):
+ super().__init__()
+ self.num_experts = original.gate_up_proj.shape[0]
+ intermediate_size = config.moe_intermediate_size
+
+ with torch.device("meta"):
+ super().__init__([Qwen3_5MoeMLP(config, intermediate_size) for _ in range(self.num_experts)])
+
+ def _materialize_weights(self, original) -> None:
+ intermediate_size = original.down_proj.shape[-1]
+ if not unsupported_meta_device(original):
+ for i in range(self.num_experts):
+ gate_up = original.gate_up_proj[i]
+ down = original.down_proj[i]
+
+ gate_proj = gate_up[:intermediate_size, :]
+ up_proj = gate_up[intermediate_size:, :]
+
+ _update_parameter(self[i].gate_proj, "weight", gate_proj.contiguous())
+ _update_parameter(self[i].up_proj, "weight", up_proj.contiguous())
+ _update_parameter(self[i].down_proj, "weight", down.contiguous())
+ del gate_up, down, gate_proj, up_proj
+ original.to_empty(device="meta") # release original experts parameters
+ clear_memory()
diff --git a/auto_round/modeling/fused_moe/qwen3_vl_moe.py b/auto_round/modeling/fused_moe/qwen3_vl_moe.py
index 9864f2b10..102b5abd9 100644
--- a/auto_round/modeling/fused_moe/qwen3_vl_moe.py
+++ b/auto_round/modeling/fused_moe/qwen3_vl_moe.py
@@ -158,9 +158,9 @@ class SequentialQwen3VLMoeTextExperts(torch.nn.ModuleList):
gate_proj = gate_up[:, :intermediate_size]
up_proj = gate_up[:, intermediate_size:]
- _update_parameter(self[i].gate_proj, "weight", gate_proj.t())
- _update_parameter(self[i].up_proj, "weight", up_proj.t())
- _update_parameter(self[i].down_proj, "weight", down.t())
+ _update_parameter(self[i].gate_proj, "weight", gate_proj.t().contiguous())
+ _update_parameter(self[i].up_proj, "weight", up_proj.t().contiguous())
+ _update_parameter(self[i].down_proj, "weight", down.t().contiguous())
del gate_up, down, gate_proj, up_proj
original.to_empty(device="meta") # release original experts parameters
clear_memory()
diff --git a/auto_round/modeling/fused_moe/replace_modules.py b/auto_round/modeling/fused_moe/replace_modules.py
index 06845e93c..ac6bb8ee2 100644
--- a/auto_round/modeling/fused_moe/replace_modules.py
+++ b/auto_round/modeling/fused_moe/replace_modules.py
@@ -25,10 +25,12 @@ from auto_round.modeling.fused_moe.moe_experts_interface import LINEAR_LOOP_IMPL
from auto_round.utils import LazyImport, dump_mem_usage, dump_memory_usage_ctx, global_state, logger
BUILTIN_MODULES = {
- "Llama4TextMoe": LazyImport("auto_round.modeling.fused_moe.llama4"),
- "GptOssMLP": LazyImport("auto_round.modeling.fused_moe.gpt_oss"),
- "Qwen3VLMoeTextSparseMoeBlock": LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe"),
- "DeepseekV2Attention": LazyImport("auto_round.modeling.fused_moe.deepseek_v2"),
+ "llama4": LazyImport("auto_round.modeling.fused_moe.llama4"),
+ "gpt_oss": LazyImport("auto_round.modeling.fused_moe.gpt_oss"),
+ "qwen3_vl_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_vl_moe"),
+ "deepseek_v2": LazyImport("auto_round.modeling.fused_moe.deepseek_v2"),
+ "qwen3_5_moe": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
+ "qwen3_5_moe_text": LazyImport("auto_round.modeling.fused_moe.qwen3_5_moe"),
}
@@ -37,19 +39,11 @@ def _has_custom_replacement_only(model: torch.nn.Module) -> bool:
If all MOE modules are covered by BUILTIN_MODULES, we can skip the linear_loop path entirely.
"""
- for _, module in model.named_modules():
- class_name = module.__class__.__name__
- # Check if this looks like an MOE experts module (has fused 3D weights)
- if (
- hasattr(module, "gate_up_proj")
- and isinstance(module.gate_up_proj, torch.nn.Parameter)
- and module.gate_up_proj.dim() == 3
- ):
- # This is a fused experts module - check if it has a custom replacement
- if class_name not in BUILTIN_MODULES and not ReplacementModuleBase.is_registered(class_name):
- # Found a fused experts module without custom replacement - need linear_loop
- return False
- return True
+ if hasattr(model, "config") and hasattr(model.config, "model_type"):
+ model_type = model.config.model_type
+ if model_type in BUILTIN_MODULES:
+ return True
+ return False
def _handle_moe_modules(model: torch.nn.Module) -> list[str]:
@@ -84,15 +78,12 @@ def _handle_moe_modules(model: torch.nn.Module) -> list[str]:
def _import_required_replacements(model: torch.nn.Module) -> None:
"""Scan model and trigger lazy imports for registered replacement modules."""
imported = set()
-
- for _, module in model.named_modules():
- class_name = module.__class__.__name__
-
- if class_name in BUILTIN_MODULES and class_name not in imported:
- # Trigger import by accessing the LazyImport object
- _ = BUILTIN_MODULES[class_name].__name__ # or any attribute
- imported.add(class_name)
- logger.debug(f"Loaded replacement module for {class_name}")
+ if hasattr(model, "config") and hasattr(model.config, "model_type"):
+ model_type = model.config.model_type
+ if model_type in BUILTIN_MODULES:
+ _ = BUILTIN_MODULES[model_type].__name__ # or any attribute
+ imported.add(model_type)
+ logger.debug(f"Loaded replacement module for {model_type}")
def _should_skip_moe_replacement(module: torch.nn.Module, model: torch.nn.Module) -> bool:
@@ -129,6 +120,7 @@ def materialize_model_(model: torch.nn.Module) -> None:
module.materialize_weights()
model.apply(_materialize_module)
+
# check if any module on meta device remains
found_meta = False
for name, param in model.named_parameters():
@@ -332,7 +324,7 @@ def apply_replacements(
if isinstance(module, ReplacementModuleBase):
continue
class_name = module.__class__.__name__
- if class_name in BUILTIN_MODULES and _should_skip_moe_replacement(module, model):
+ if ReplacementModuleBase.is_registered(class_name) and _should_skip_moe_replacement(module, model):
logger.debug(f"Skipping replacement for {name}: linear_loop experts already unfused")
continue
if ReplacementModuleBase.is_registered(class_name) and ReplacementModuleBase.get_replacement_class(
diff --git a/auto_round/special_model_handler.py b/auto_round/special_model_handler.py
index 7627f0782..2371941f5 100644
--- a/auto_round/special_model_handler.py
+++ b/auto_round/special_model_handler.py
@@ -217,16 +217,34 @@ def register_ignore_layers(
_PRE_DEFINED_IGNORE_LAYERS.append(rule)
-# Qwen3MOE
register_ignore_layers(
matchers=[
- ArchitectureMatcher(r"Qwen3.*Moe", mode="regex"),
+ ModelTypeMatcher(r"qwen3_vl_moe", mode="full"),
],
ignore_layers=[
"mlp.gate", # vllm inference issue
],
)
+register_ignore_layers(
+ matchers=[
+ ModelTypeMatcher(r"qwen3_moe", mode="full"),
+ ],
+ ignore_layers=[
+ "mlp.gate", # vllm inference issue
+ ],
+)
+
+register_ignore_layers(
+ matchers=[
+ ModelTypeMatcher(r"qwen3_5_moe", mode="full"),
+ ],
+ ignore_layers=[
+ "mlp.gate", # vllm inference issue
+ ],
+)
+
+
# longcat
register_ignore_layers(
matchers=[
@@ -266,17 +284,6 @@ register_ignore_layers(
)
-# # qwen3_next
-# register_ignore_layers(
-# matchers=[
-# ModelTypeMatcher(r"qwen3_next", mode="full"),
-# ],
-# ignore_layers=[
-# "mlp.gate", # vllm issue
-# ],
-# )
-
-
def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:
layers = []
for rule in _PRE_DEFINED_IGNORE_LAYERS:
@@ -291,5 +298,10 @@ def get_predefined_ignore_layers(model: torch.nn.Module) -> list[str]:
else:
layers.extend(res)
break
+ if not layers:
+ if hasattr(model, "config") and hasattr(model.config, "model_type"):
+ model_type = model.config.model_type
+ if "moe" in model_type: # Append gate which usually cause vllm issue
+ layers.append("mlp.gate")
return list(dict.fromkeys(layers))
diff --git a/auto_round/utils/model.py b/auto_round/utils/model.py
index 1442512c9..3d010d150 100644
--- a/auto_round/utils/model.py
+++ b/auto_round/utils/model.py
@@ -1426,7 +1426,6 @@ def _get_reference_amax_from_experts(moe_module: torch.nn.Module, attr_name: str
# 5b3ddff74cae9651f24bef15d3255c4ee053fc60/src/llmcompressor/pytorch/model_load/helpers.py#L144
def copy_python_files_from_model_cache(model, save_path: str):
config = model.config
- cache_path = None
if hasattr(config, "_name_or_path"):
import os
import shutil
@@ -1434,14 +1433,13 @@ def copy_python_files_from_model_cache(model, save_path: str):
from huggingface_hub import hf_hub_download
if version.parse(transformers.__version__) < version.parse("5.0.0"):
- from transformers import TRANSFORMERS_CACHE
+ from transformers.utils import TRANSFORMERS_CACHE
- cache_dir = TRANSFORMERS_CACHE
+ cache_dir = os.environ.get("HF_HOME", TRANSFORMERS_CACHE)
+ else:
from huggingface_hub.constants import HF_HUB_CACHE
- cache_dir = os.environ.get("HF_HOME") or HF_HUB_CACHE
-
- cache_dir = os.environ.get("HF_HOME", None)
+ cache_dir = os.environ.get("HF_HOME", HF_HUB_CACHE)
from transformers.utils import http_user_agent
cache_path = config._name_or_path
diff --git a/test/helpers.py b/test/helpers.py
index 8558fbd1b..2f98e4e55 100644
--- a/test/helpers.py
+++ b/test/helpers.py
@@ -242,6 +242,7 @@ def get_output(model_name_or_path):
return outputs.detach().cpu()
def is_model_outputs_similar(model_path_1, model_path_2, metric="cosine_similarity", threshold=0.98, k=5, verbose=True):
"""
Compare outputs from two models using specified metric and return pass/fail.
diff --git a/test/test_cpu/schemes/test_scheme.py b/test/test_cpu/schemes/test_scheme.py
index dcafb3ca8..77ab5fd1f 100644
--- a/test/test_cpu/schemes/test_scheme.py
+++ b/test/test_cpu/schemes/test_scheme.py
@@ -61,7 +61,7 @@ class TestAutoRound:
assert ar.model.model.layers[0].self_attn.q_proj.bits == 8
assert ar.model.model.layers[0].self_attn.k_proj.bits == 16
assert ar.model.model.layers[0].mlp.experts[0].up_proj.bits == 4
- assert ar.model.model.layers[0].mlp.shared_expert.gate_proj.bits == 8
+ # assert ar.model.model.layers[0].mlp.shared_expert.gate_proj.bits == 8 # gate has been added to ignore_layers
shutil.rmtree(self.save_folder, ignore_errors=True)