PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
gguf format add support for mtp quantization (#1866)
Heng Guof7e77e034d4f
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex a8a7bc5ab..f970d89f9 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -554,8 +554,9 @@ class BaseCompressor(object):is_gguf_format = _formats is not None and any("gguf" in str(getattr(fmt, "output_format", "")) for fmt in _formats)- predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)+ predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model) if not is_gguf_format else []compressed_predefined_ignore_layers = compress_layer_names(predefined_ignore_layers)+if not is_gguf_format:predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)if predefined_ignore_layers and self.quant_block_list:@@ -1063,7 +1064,15 @@ class BaseCompressor(object):self._offloader.reset()# Disable inplace when quantized layers live outside transformer blocks.- if self.has_qlayer_outside_block and self.need_calib:+ # gguf lm-head used rtn in version>=0.13+ if (+ self.has_qlayer_outside_block+ and self.need_calib+ and (+ self.compress_context.formats is None+ or "gguf" not in self.compress_context.formats[0].__class__.__name__.lower()+ )+ ):self.inplace = Falseif not hasattr(self, "formats"):@@ -1222,12 +1231,14 @@ class BaseCompressor(object):returnformats = getattr(self, "formats", [])- if len(formats) == 1 and not formats[0].is_fake() and self.inplace:+ has_single_gguf_format = len(formats) == 1 and formats[0].is_gguf()+ # GGUF supports per-block / per-layer immediate packing even when+ # full-model in-place rewriting is disabled by outside-block layers.+ if len(formats) == 1 and not formats[0].is_fake() and (self.inplace or has_single_gguf_format):self.compress_context.is_immediate_packing = True- if self.has_qlayer_outside_block and self.need_calib:+ if self.has_qlayer_outside_block and self.need_calib and not has_single_gguf_format:self.compress_context.is_immediate_packing = False-if not ("causallm" in self.model_context.model.__class__.__name__.lower() and not self.model_context.is_mllm):# TODO For tied keys, there may some issues, we haven't not verified thistied_weight_keys = getattr(self.model_context.model, "_tied_weight_keys", {})diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 1ef4e5720..3050b62eb 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -26,6 +26,7 @@ import transformersfrom torch.amp import autocastfrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType+from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector, gguf_format_to_ftypefrom auto_round.logger import loggerfrom auto_round.utils import (check_to_quantized,@@ -774,25 +775,36 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, modeltie_word_embeddings = model.config.tie_word_embeddingstie_word_embeddings &= not is_separate_lm_head(model)- n_gqa = 1- if (- hasattr(model, "config")- and hasattr(model.config, "num_attention_heads")- and hasattr(model.config, "num_key_value_heads")- ):- n_gqa = model.config.num_attention_heads // model.config.num_key_value_heads- n_expert = 0- for name in ["num_experts", "num_local_experts", "n_routed_experts"]:- if hasattr(model.config, name):- n_expert = getattr(model.config, name)-- i_attention_wv = 0- i_ffn_down = 0+ dtype_selector = GGUFDTypeSelector(+ hparams, gguf_format_to_ftype(target_gguf_format), model_class.model_arch, n_layer+ )layer_config_copy = copy.deepcopy(layer_config)base_target_bits = Noneif inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():base_target_bits = int(inner_gguf_format[6])+ def _resolve_gguf_name(layer_name):+ if model_type != ModelType.TEXT and any([key in layer_name for key in MM_MODULE_KEYS]):+ gguf_layer_name = tensor_map_vision.get_name(layer_name)+ if gguf_layer_name is None:+ for key in MM_MODULE_KEYS:+ gguf_layer_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))+ if gguf_layer_name is not None:+ break+ else:+ gguf_layer_name = tensor_map.get_name(layer_name)+ if gguf_layer_name is None:+ gguf_layer_name = tensor_map.get_name(layer_name.replace(".language_model", ""))+ return gguf_layer_name++ dtype_selector.n_attention_wv = sum(+ 1+ for layer_name, config in layer_config_copy.items()+ if check_to_quantized(config)+ and (gguf_name := _resolve_gguf_name(layer_name))+ and any(key in gguf_name for key in ("attn_v", "attn_qkv", "attn_kv_b"))+ )+for layer_name, config in layer_config_copy.items():if not check_to_quantized(config):continue@@ -815,17 +827,7 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, modelre.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format][embedding_format_key]).group(1))- if model_type != ModelType.TEXT and any([key in layer_name for key in MM_MODULE_KEYS]):- gguf_name = tensor_map_vision.get_name(layer_name)- if gguf_name is None:- for key in MM_MODULE_KEYS:- gguf_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))- if gguf_name is not None:- break- else:- gguf_name = tensor_map.get_name(layer_name)- if gguf_name is None:- gguf_name = tensor_map.get_name(layer_name.replace(".language_model", ""))+ gguf_name = _resolve_gguf_name(layer_name)bits_index = 6if config.get("fixed_by_user", False):if "bits" not in config:@@ -877,116 +879,9 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, modelnew_type = _search_gguf_type(new_type)if new_type is None:raise ValueError(f"invalid bit setting for {layer_name}")- elif gguf_name is None:- pass- # attn_v- elif "attn_v" in gguf_name:- if target_gguf_format == "gguf:q2_k":- new_type = "gguf:q4_k" if n_gqa >= 4 else "gguf:q3_k"- elif target_gguf_format == "gguf:q2_k_s" and n_gqa >= 4:- new_type = "gguf:q4_k"- elif target_gguf_format == "gguf:q3_k_m":- new_type = "gguf:q5_k" if i_attention_wv < 2 else "gguf:q4_k"- elif target_gguf_format == "gguf:q3_k_l":- new_type = "gguf:q5_k"- elif (target_gguf_format == "gguf:q4_k_m" or target_gguf_format == "gguf:q5_k_m") and _use_more_bits(- i_layer, n_layer- ):- new_type = "gguf:q6_k"- elif target_gguf_format == "gguf:q4_k_s" and i_attention_wv < 4:- new_type = "gguf:q5_k"- ##TODO check which models are be grouped into to LLM_TYPE_70B- # if (qs.model.type == LLM_TYPE_70B) {- # // In the 70B model we have 8 heads sharing the same attn_v weights.- # As a result, the attn_v.weight tensor is- # // 8x smaller compared to attn_q.weight.Hence, we can get a nice boost in quantization accuracy with- # // nearly negligible increase in model size by quantizing this tensor with more bits:- # if- # (new_type == GGML_TYPE_Q3_K | | new_type == GGML_TYPE_Q4_K)- # new_type = GGML_TYPE_Q5_K;- # }- if n_expert == 8:- new_type = "gguf:q8_k"- i_attention_wv += 1-- elif "attn_k" in gguf_name:- if n_expert == 8:- new_type = "gguf:q8_0"- # ffn_down- elif "ffn_down" in gguf_name:- if target_gguf_format == "gguf:q2_k":- new_type = "gguf:q3_k"- elif target_gguf_format == "gguf:q2_k_s":- if i_layer < n_layer / 8:- new_type = "gguf:q4_k"- elif target_gguf_format == "gguf:q3_k_m":- if i_layer < n_layer / 16:- new_type = "gguf:q5_k"- elif gguf.MODEL_ARCH.FALCON == model_class.model_arch or _use_more_bits(i_layer, n_layer):- new_type = "gguf:q4_k"- else:- new_type = "gguf:q3_k"- elif target_gguf_format == "gguf:q3_k_l":- if gguf.MODEL_ARCH.FALCON == model_class.model_arch:- new_type = "gguf:q4_k"- else:- new_type = "gguf:q5_k"- elif target_gguf_format == "gguf:q4_k_m":- if gguf.MODEL_ARCH.FALCON == model_class.model_arch:- if i_layer < n_layer // 16:- new_type = "gguf:q6_k"- elif _use_more_bits(i_layer, n_layer):- new_type = "gguf:q5_k"- else:- new_type = "gguf:q4_k"- else:- if _use_more_bits(i_layer, n_layer):- new_type = "gguf:q6_k"- elif target_gguf_format == "gguf:q5_k_m" and _use_more_bits(i_layer, n_layer):- new_type = "gguf:q6_k"- elif (- target_gguf_format == "gguf:q4_k_s"- and model_class.model_arch != gguf.MODEL_ARCH.FALCON- and i_layer < n_layer / 8- ):- new_type = "gguf:q5_k"- elif (target_gguf_format == "gguf:q4_0" or target_gguf_format == "gguf:q5_0") and i_layer < n_layer / 8:- if target_gguf_format == "gguf:q4_0":- new_type = "gguf:q4_1"- else:- new_type = "gguf:q5_1"- i_ffn_down += 1-- # attn_output- elif "attn_output" in gguf_name:- if gguf.MODEL_ARCH.FALCON != model_class.model_arch:- if n_expert == 8:- if target_gguf_format in (- "gguf:q2_k",- "gguf:q3_k_s",- "gguf:q3_k_m",- "gguf:q4_k_s",- "gguf:q4_k_m",- "gguf:q5_k",- ):- new_type = "gguf:q5_k"- elif target_gguf_format == "gguf:q2_k":- new_type = "gguf:q3_k"- elif target_gguf_format == "gguf:q3_k_m":- new_type = "gguf:q4_k"- elif target_gguf_format == "gguf:q3_k_l":- new_type = "gguf:q5_k"- else:- if target_gguf_format == "gguf:q3_k_l":- new_type = "gguf:q4_k"- # attn_qkv- elif "attn_qkv" in gguf_name:- if target_gguf_format in ("gguf:q3_k_m", "gguf:q3_k_l"):- new_type = "gguf:q4_k"- elif target_gguf_format == "gguf:q4_k_m":- new_type = "gguf:q5_k"- elif target_gguf_format == "gguf:q5_k_m":- new_type = "gguf:q5_k"+ elif gguf_name is not None:+ gguf_weight_name = gguf_name if gguf_name.endswith(".weight") else f"{gguf_name}.weight"+ new_type = dtype_selector.select_gguf_type(gguf_weight_name, len(layer.weight.shape), i_layer or 0)new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]if input_features % new_block_size != 0:new_type = _gguf_type_fallback(new_type)diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 0af48ce61..539c90c00 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -47,6 +47,7 @@ import torchfrom transformers import AutoConfigfrom auto_round.export.export_to_gguf.config import ModelType+from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelectorfrom auto_round.export.export_to_gguf.packing import ggml_quantfrom auto_round.utils import (LazyImport,@@ -297,7 +298,7 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, dreturn data, data_qtype-def get_qtype_by_layer_config(layer_config, name, data_qtype):+def get_qtype_by_layer_config(layer_config, name, data_qtype, *, explicit_only=False):name = name[: -len(".weight")]if name not in layer_config and name.endswith("embed_tokens"):embedding_names = [key for key in layer_config if key.endswith("embed_tokens")]@@ -307,7 +308,9 @@ def get_qtype_by_layer_config(layer_config, name, data_qtype):embedding_names = [key for key in layer_config if key.endswith("embed_tokens")]if len(embedding_names) == 1:name = embedding_names[0]- if name not in layer_config or layer_config[name]["bits"] >= 16:+ if name not in layer_config:+ return None if explicit_only else data_qtype+ if layer_config[name]["bits"] >= 16:return data_qtypebits = layer_config[name].get("bits")super_bits = layer_config[name].get("super_bits")@@ -413,8 +416,28 @@ def _special_name_handle(cls, name):return name+def _count_attention_wv_tensors(cls):+ count = 0+ for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):+ if data_torch is None or data_torch.numel() == 0:+ continue+ filtered = cls.filter_tensors((name, lambda data_torch=data_torch: data_torch))+ if filtered is None:+ continue+ modify_name = _special_name_handle(cls, filtered[0])+ try:+ new_name = cls.map_tensor_name(modify_name)+ except Exception:+ continue+ count += any(key in new_name for key in ("attn_v.weight", "attn_qkv.weight", "attn_kv_b.weight"))+ return count or None++def prepare_tensors(cls):device = get_packing_device(cls.device)+ if not hasattr(cls, "_gguf_dtype_selector"):+ cls._gguf_dtype_selector = GGUFDTypeSelector(cls.hparams, cls.ftype, cls.model_arch)+ cls._gguf_dtype_selector.n_attention_wv = _count_attention_wv_tensors(cls)# Handle empty tensor_map for models with block_count=0 (like MobileNetV5)if cls.tensor_map.mapping:@@ -463,7 +486,6 @@ def prepare_tensors(cls):clean_weight_list = []modify_name = _special_name_handle(cls, name)- data_torch = data_torch.to("cpu")for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):skip = Falsefor tensor_info in cls.gguf_writer.tensors:@@ -539,8 +561,13 @@ def prepare_tensors(cls):name = get_moe_name(cls, name, new_name)clean_weight_list.append(name)# get data_qtype by layer_config- data_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype)+ layer_config_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype, explicit_only=True)# # No override (data_qtype is False), or wants to be quantized (data_qtype is True)+ if layer_config_qtype is not None:+ data_qtype = layer_config_qtype+ elif isinstance(data_qtype, bool):+ data_qtype = cls._gguf_dtype_selector.select_qtype(new_name, n_dims)+if isinstance(data_qtype, bool):if cls.ftype == gguf.LlamaFileType.ALL_F32:data_qtype = gguf.GGMLQuantizationType.F32@@ -611,7 +638,7 @@ def prepare_tensors(cls):# if data ends up empty, it means data_torch was a scalar tensor -> restoreif len(data_torch.shape) == 0:- data = data_torch.numpy()+ data = data_torch.cpu().numpy()try:data = gguf.quants.quantize(data, data_qtype)except gguf.QuantError as e:diff --git a/auto_round/export/export_to_gguf/gguf_dtype.py b/auto_round/export/export_to_gguf/gguf_dtype.pynew file mode 100644index 000000000..93c9f2ff9--- /dev/null+++ b/auto_round/export/export_to_gguf/gguf_dtype.py@@ -0,0 +1,320 @@+# Copyright (c) 2026 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from __future__ import annotations++from enum import Enum++from auto_round.utils import LazyImport++gguf = LazyImport("gguf")+++_GGUF_TYPE_TO_QTYPE_NAME = {+ "gguf:fp16": "F16",+ "gguf:f16": "F16",+ "gguf:bf16": "BF16",+ "gguf:f32": "F32",+ "gguf:q2_k": "Q2_K",+ "gguf:q3_k": "Q3_K",+ "gguf:q4_k": "Q4_K",+ "gguf:q5_k": "Q5_K",+ "gguf:q6_k": "Q6_K",+ "gguf:q4_0": "Q4_0",+ "gguf:q4_1": "Q4_1",+ "gguf:q5_0": "Q5_0",+ "gguf:q5_1": "Q5_1",+ "gguf:q8_0": "Q8_0",+}+++_QTYPE_NAME_TO_GGUF_TYPE = {qtype: gguf_type for gguf_type, qtype in _GGUF_TYPE_TO_QTYPE_NAME.items()}+_QTYPE_NAME_TO_GGUF_TYPE["F16"] = "gguf:fp16"+++class TensorCategory(Enum):+ TOKEN_EMBD = "token_embd"+ ATTENTION_Q = "attn_q"+ ATTENTION_V = "attn_v"+ ATTENTION_K = "attn_k"+ ATTENTION_QKV = "attn_qkv"+ ATTENTION_KV_B = "attn_kv_b"+ ATTENTION_OUTPUT = "attn_output"+ FFN_UP = "ffn_up"+ FFN_GATE = "ffn_gate"+ FFN_DOWN = "ffn_down"+ OUTPUT = "output"+ OTHER = "other"+++def _tensor_category(name: str) -> TensorCategory:+ if name == "output.weight":+ return TensorCategory.OUTPUT+ if name in ("token_embd.weight", "per_layer_token_embd.weight"):+ return TensorCategory.TOKEN_EMBD+ if "attn_qkv.weight" in name:+ return TensorCategory.ATTENTION_QKV+ if "attn_kv_b.weight" in name:+ return TensorCategory.ATTENTION_KV_B+ if "attn_v.weight" in name:+ return TensorCategory.ATTENTION_V+ if "attn_k.weight" in name:+ return TensorCategory.ATTENTION_K+ if "attn_q.weight" in name:+ return TensorCategory.ATTENTION_Q+ if "attn_output.weight" in name:+ return TensorCategory.ATTENTION_OUTPUT+ if "ffn_up" in name:+ return TensorCategory.FFN_UP+ if "ffn_gate" in name:+ return TensorCategory.FFN_GATE+ if "ffn_down" in name:+ return TensorCategory.FFN_DOWN+ return TensorCategory.OTHER+++def _is_attn_v_like(category: TensorCategory) -> bool:+ return category in (TensorCategory.ATTENTION_V, TensorCategory.ATTENTION_QKV, TensorCategory.ATTENTION_KV_B)+++def _use_more_bits(i_layer: int, n_layer: int) -> bool:+ return i_layer < n_layer // 8 or i_layer >= 7 * n_layer // 8 or (i_layer - n_layer // 8) % 3 == 2+++def _get_layer_id(name: str, fallback: int) -> int:+ parts = name.split(".")+ if len(parts) > 1 and parts[0] == "blk" and parts[1].isdigit():+ return int(parts[1])+ return fallback+++def _default_qtype(ftype):+ if ftype == gguf.LlamaFileType.ALL_F32:+ return gguf.GGMLQuantizationType.F32+ if ftype == gguf.LlamaFileType.MOSTLY_F16:+ return gguf.GGMLQuantizationType.F16+ if ftype == gguf.LlamaFileType.MOSTLY_BF16:+ return gguf.GGMLQuantizationType.BF16+ if ftype == gguf.LlamaFileType.MOSTLY_Q8_0:+ return gguf.GGMLQuantizationType.Q8_0+ if ftype == gguf.LlamaFileType.MOSTLY_Q4_0:+ return gguf.GGMLQuantizationType.Q4_0+ if ftype == gguf.LlamaFileType.MOSTLY_Q4_1:+ return gguf.GGMLQuantizationType.Q4_1+ if ftype == gguf.LlamaFileType.MOSTLY_Q5_0:+ return gguf.GGMLQuantizationType.Q5_0+ if ftype == gguf.LlamaFileType.MOSTLY_Q5_1:+ return gguf.GGMLQuantizationType.Q5_1+ if ftype in (gguf.LlamaFileType.MOSTLY_Q2_K_S, gguf.LlamaFileType.MOSTLY_Q2_K):+ return gguf.GGMLQuantizationType.Q2_K+ if ftype in (+ gguf.LlamaFileType.MOSTLY_Q3_K_S,+ gguf.LlamaFileType.MOSTLY_Q3_K_M,+ gguf.LlamaFileType.MOSTLY_Q3_K_L,+ ):+ return gguf.GGMLQuantizationType.Q3_K+ if ftype in (gguf.LlamaFileType.MOSTLY_Q4_K_S, gguf.LlamaFileType.MOSTLY_Q4_K_M):+ return gguf.GGMLQuantizationType.Q4_K+ if ftype in (gguf.LlamaFileType.MOSTLY_Q5_K_S, gguf.LlamaFileType.MOSTLY_Q5_K_M):+ return gguf.GGMLQuantizationType.Q5_K+ if ftype == gguf.LlamaFileType.MOSTLY_Q6_K:+ return gguf.GGMLQuantizationType.Q6_K+ raise ValueError(f"Unknown file type: {ftype.name}")+++def gguf_format_to_ftype(gguf_format: str):+ format_name = gguf_format.lower()+ if format_name == "gguf:q2_k_mixed":+ format_name = "gguf:q2_k_s"+ mapping = {+ "gguf:f32": gguf.LlamaFileType.ALL_F32,+ "gguf:fp16": gguf.LlamaFileType.MOSTLY_F16,+ "gguf:f16": gguf.LlamaFileType.MOSTLY_F16,+ "gguf:bf16": gguf.LlamaFileType.MOSTLY_BF16,+ "gguf:q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,+ "gguf:q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,+ "gguf:q5_0": gguf.LlamaFileType.MOSTLY_Q5_0,+ "gguf:q5_1": gguf.LlamaFileType.MOSTLY_Q5_1,+ "gguf:q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,+ "gguf:q2_k": gguf.LlamaFileType.MOSTLY_Q2_K,+ "gguf:q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,+ "gguf:q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,+ "gguf:q3_k_m": gguf.LlamaFileType.MOSTLY_Q3_K_M,+ "gguf:q3_k_l": gguf.LlamaFileType.MOSTLY_Q3_K_L,+ "gguf:q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,+ "gguf:q4_k_m": gguf.LlamaFileType.MOSTLY_Q4_K_M,+ "gguf:q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,+ "gguf:q5_k_m": gguf.LlamaFileType.MOSTLY_Q5_K_M,+ "gguf:q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,+ "gguf:q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,+ }+ if format_name not in mapping:+ raise ValueError(f"Unknown GGUF format: {gguf_format}")+ return mapping[format_name]+++def qtype_to_gguf_type(qtype) -> str:+ return _QTYPE_NAME_TO_GGUF_TYPE[qtype.name]+++class GGUFDTypeSelector:+ def __init__(+ self,+ hparams: dict,+ ftype,+ model_arch=None,+ n_layer: int | None = None,+ n_attention_wv: int | None = None,+ has_imatrix: bool = False,+ ):+ self.hparams = hparams+ self.ftype = ftype+ self.model_arch = model_arch+ self.n_layer = n_layer+ self.n_attention_wv = n_attention_wv+ self.has_imatrix = has_imatrix+ self.i_attention_wv = 0+ self.i_ffn_down = 0++ def select_qtype(self, name: str, n_dims: int, fallback_index: int = 0):+ """Select the GGML tensor dtype used by llama.cpp quantization.++ `name` must be the final GGUF tensor name. MTP/NextN tensors are already+ remapped to extended `blk.<id>` names by conversion classes, so they use+ the same category and layer-index rules as regular transformer blocks.+ """+ if n_dims < 2:+ return gguf.GGMLQuantizationType.F32++ qtype = _default_qtype(self.ftype)+ category = _tensor_category(name)+ i_layer = _get_layer_id(name, fallback_index)+ if self.n_layer is None:+ n_layer = int(+ self.hparams.get("num_hidden_layers", self.hparams.get("n_layer", self.hparams.get("num_layers", 1)))+ )+ n_layer += int(+ self.hparams.get("mtp_num_hidden_layers", self.hparams.get("num_nextn_predict_layers", 0)) or 0+ )+ else:+ n_layer = self.n_layer+ n_layer = max(n_layer, i_layer + 1)+ n_gqa = 1+ n_head = self.hparams.get("num_attention_heads", self.hparams.get("n_head"))+ n_head_kv = self.hparams.get("num_key_value_heads", self.hparams.get("n_head_kv"))+ if n_head and n_head_kv:+ n_gqa = max(1, int(n_head) // int(n_head_kv))+ n_expert = int(+ self.hparams.get(+ "num_experts", self.hparams.get("num_local_experts", self.hparams.get("n_routed_experts", 0))+ )+ or 0+ )++ if _is_attn_v_like(category):+ if self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:+ qtype = gguf.GGMLQuantizationType.Q4_K if n_gqa >= 4 else gguf.GGMLQuantizationType.Q3_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S and n_gqa >= 4:+ qtype = gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M:+ qtype = gguf.GGMLQuantizationType.Q5_K if self.i_attention_wv < 2 else gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:+ qtype = gguf.GGMLQuantizationType.Q5_K+ elif self.ftype in (gguf.LlamaFileType.MOSTLY_Q4_K_M, gguf.LlamaFileType.MOSTLY_Q5_K_M) and _use_more_bits(+ self.i_attention_wv, self.n_attention_wv or n_layer+ ):+ qtype = gguf.GGMLQuantizationType.Q6_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S and self.i_attention_wv < 4:+ qtype = gguf.GGMLQuantizationType.Q5_K+ if n_expert == 8:+ qtype = gguf.GGMLQuantizationType.Q8_0+ self.i_attention_wv += 1+ elif category == TensorCategory.ATTENTION_K and n_expert == 8:+ qtype = gguf.GGMLQuantizationType.Q8_0+ elif category == TensorCategory.FFN_DOWN:+ if self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:+ qtype = gguf.GGMLQuantizationType.Q3_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S and i_layer < n_layer // 8:+ qtype = gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M:+ if i_layer < n_layer // 16:+ qtype = gguf.GGMLQuantizationType.Q5_K+ elif self.model_arch == gguf.MODEL_ARCH.FALCON or _use_more_bits(i_layer, n_layer):+ qtype = gguf.GGMLQuantizationType.Q4_K+ else:+ qtype = gguf.GGMLQuantizationType.Q3_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:+ qtype = (+ gguf.GGMLQuantizationType.Q4_K+ if self.model_arch == gguf.MODEL_ARCH.FALCON+ else gguf.GGMLQuantizationType.Q5_K+ )+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:+ if self.model_arch == gguf.MODEL_ARCH.FALCON:+ if i_layer < n_layer // 16:+ qtype = gguf.GGMLQuantizationType.Q6_K+ elif _use_more_bits(i_layer, n_layer):+ qtype = gguf.GGMLQuantizationType.Q5_K+ elif _use_more_bits(i_layer, n_layer):+ qtype = gguf.GGMLQuantizationType.Q6_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M and _use_more_bits(i_layer, n_layer):+ qtype = gguf.GGMLQuantizationType.Q6_K+ elif (+ self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S+ and self.model_arch != gguf.MODEL_ARCH.FALCON+ and i_layer < n_layer // 8+ ):+ qtype = gguf.GGMLQuantizationType.Q5_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0 and self.has_imatrix and i_layer < n_layer // 8:+ qtype = gguf.GGMLQuantizationType.Q4_1+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0 and self.has_imatrix and i_layer < n_layer // 8:+ qtype = gguf.GGMLQuantizationType.Q5_1+ self.i_ffn_down += 1+ elif category == TensorCategory.ATTENTION_OUTPUT:+ if self.model_arch != gguf.MODEL_ARCH.FALCON:+ if n_expert == 8:+ if self.ftype in (+ gguf.LlamaFileType.MOSTLY_Q2_K,+ gguf.LlamaFileType.MOSTLY_Q3_K_S,+ gguf.LlamaFileType.MOSTLY_Q3_K_M,+ gguf.LlamaFileType.MOSTLY_Q4_K_S,+ gguf.LlamaFileType.MOSTLY_Q4_K_M,+ ):+ qtype = gguf.GGMLQuantizationType.Q5_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:+ qtype = gguf.GGMLQuantizationType.Q3_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M:+ qtype = gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:+ qtype = gguf.GGMLQuantizationType.Q5_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:+ qtype = gguf.GGMLQuantizationType.Q4_K+ elif category == TensorCategory.ATTENTION_QKV:+ if self.ftype in (gguf.LlamaFileType.MOSTLY_Q3_K_M, gguf.LlamaFileType.MOSTLY_Q3_K_L):+ qtype = gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:+ qtype = gguf.GGMLQuantizationType.Q5_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M:+ qtype = gguf.GGMLQuantizationType.Q6_K++ return qtype++ def select_gguf_type(self, name: str, n_dims: int, fallback_index: int = 0) -> str:+ return qtype_to_gguf_type(self.select_qtype(name, n_dims, fallback_index=fallback_index))+++def select_llama_cpp_compatible_qtype(name: str, ftype, hparams: dict, n_dims: int, fallback_index: int = 0):+ selector = GGUFDTypeSelector(hparams, ftype)+ selector.i_attention_wv = fallback_index+ return selector.select_qtype(name, n_dims, fallback_index=fallback_index)diff --git a/auto_round/formats.py b/auto_round/formats.pyindex 80313e824..57b6e4f3c 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -780,9 +780,19 @@ class GGUFFormat(OutputFormat):gguf_format = f"gguf:{format.lower()}"if format.lower().endswith("_mixed"):from auto_round.schemes import _handle_special_schemes+ from auto_round.utils.model import is_moe_model- ar.layer_config = _handle_special_schemes(gguf_format, ar.layer_config, ar.model)- gguf_format = gguf_format.lower().replace("_mixed", "_s")+ if format.lower() == "q2_k_mixed" and getattr(ar, "iters", 0) > 0 and not is_moe_model(ar.model):+ logger.warning(+ "gguf:q2_k_mixed only supports MoE models with iters>0. "+ "It is not an MoE model, falling back to gguf:q4_k_m."+ )+ gguf_format = "gguf:q4_k_m"+ else:+ ar.layer_config = _handle_special_schemes(+ gguf_format, ar.layer_config, ar.model, quant_nontext_module=ar.quant_nontext_module+ )+ gguf_format = gguf_format.lower().replace("_mixed", "_s")if isinstance(scheme, str) and scheme.lower() != gguf_format:logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")ar.scheme = gguf_formatdiff --git a/auto_round/schemes.py b/auto_round/schemes.pyindex a298575aa..d351a3f07 100644--- a/auto_round/schemes.py+++ b/auto_round/schemes.py@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, Optional, Unionimport torchfrom auto_round.logger import logger-from auto_round.utils import SUPPORTED_DTYPES, infer_bits_by_data_type+from auto_round.utils import SUPPORTED_DTYPES, contain_any_mm_keys, infer_bits_by_data_type__all__ = ["QuantizationScheme", "get_gguf_scheme", "preset_name_to_scheme"]@@ -576,6 +576,7 @@ def _handle_special_schemes(inner_supported_types=None,quant_lm_head=False,mllm=False,+ quant_nontext_module=False,) -> dict:"""handle special schemes, like GGUF:Q2_K_MIXED.Provide some special auto_round recipes.@@ -589,10 +590,16 @@ def _handle_special_schemes(for n, m in model.named_modules():if n in layer_config:continue+ if isinstance(m, torch.nn.Linear) and contain_any_mm_keys(n) and not quant_nontext_module:+ layer_config[n] = "BF16"+ continueif n == "lm_head" or isinstance(m, torch.nn.Embedding):layer_config[n] = "GGUF:Q8_0"- elif isinstance(m, torch.nn.Linear) and ("expert" not in n or "shared_experts" in n) and n != "lm_head":- layer_config[n] = "GGUF:Q4_K_S"+ elif isinstance(m, torch.nn.Linear) and ("expert" not in n or "shared_expert" in n) and n != "lm_head":+ if m.weight.ndim == 3:+ layer_config[n] = "GGUF:Q2_K_S"+ else:+ layer_config[n] = "GGUF:Q4_K_S"if scheme_name.lower() == "w4a16_mixed":logger.warning("W4A16_MIXED is experimental and the recipe may change in the future.")from auto_round.utils import get_lm_head_name@@ -609,6 +616,13 @@ def _handle_special_schemes(for n, m in model.named_modules():if n in layer_config:continue+ if (+ (type(m) in supported_types or type(m) in inner_supported_types)+ and contain_any_mm_keys(n)+ and quant_nontext_module+ ):+ layer_config[n] = {"bits": 16}+ continueif type(m) in supported_types or type(m) in inner_supported_types:if "expert" in n and "shared" not in n:layer_config[n] = {"bits": 4, "data_type": "int"}diff --git a/auto_round/utils/common.py b/auto_round/utils/common.pyindex c8b7691cf..5e7a50e6a 100644--- a/auto_round/utils/common.py+++ b/auto_round/utils/common.py@@ -688,6 +688,13 @@ MM_MODULE_KEYS = [MM_KEYS = [*MM_MODULE_KEYS, "speech", "wav", "waveform"]+def contain_any_mm_keys(name: str) -> bool:+ for key in MM_MODULE_KEYS:+ if key in name:+ return True+ return False++def is_debug_mode():"""Checks if the Python interpreter is running in debug mode.diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex cb76df646..56daf87f9 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -142,7 +142,7 @@ class TestGGUF:assert False, "cmd line test fail, please have a check"shutil.rmtree("../../tmp_autoround", ignore_errors=True)- # test mixed q2_k_s+ # test q2_k_mixed with iters=0 (RTN) on non-MoE model — should still workres = os.system(f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {model_name}"f" --bs 16 --iters 0 --disable_opt_rtn --nsamples 1 --seqlen 16 --scheme GGUF:Q2_K_MIXED"@@ -151,6 +151,15 @@ class TestGGUF:assert False, "cmd line test fail, please have a check"shutil.rmtree("../../tmp_autoround", ignore_errors=True)+ # test q2_k_mixed with iters=1 on non-MoE model — should fallback to q4_k_m+ res = os.system(+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {model_name}"+ f" --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format gguf:q2_k_mixed"+ )+ if res > 0 or res == -1:+ assert False, "cmd line test fail, please have a check"+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)+def test_vlm_gguf(self, tiny_qwen_vl_model_path):from auto_round import AutoRound@@ -313,3 +322,25 @@ class TestGGUF:assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"assert gguf_model.get_tensor(9).name == "blk.0.ffn_up_exps.weight"assert gguf_model.get_tensor(9).tensor_type.name == "Q2_K"++ def test_q2k_mixed_keeps_only_three_dim_expert_weights_at_q2k(self, tiny_qwen_moe_model_path):+ model_name = tiny_qwen_moe_model_path+ autoround = AutoRound(+ model_name,+ iters=0,+ nsamples=1,+ seqlen=16,+ disable_opt_rtn=True,+ )+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q2_k_mixed")+ gguf_file = os.listdir(quantized_model_path)[0]+ from gguf.gguf_reader import GGUFReader++ gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))+ tensor_types = {tensor.name: tensor.tensor_type.name for tensor in gguf_model.tensors}++ assert tensor_types["token_embd.weight"] == "Q8_0"+ assert tensor_types["output.weight"] == "Q8_0"+ assert tensor_types["blk.0.attn_k.weight"] == "Q4_K"+ assert tensor_types["blk.0.ffn_down_exps.weight"] == "Q5_0"+ assert tensor_types["blk.0.ffn_up_exps.weight"] == "Q2_K"diff --git a/test/test_cpu/export/test_gguf_mtp_dtype.py b/test/test_cpu/export/test_gguf_mtp_dtype.pynew file mode 100644index 000000000..a036b8a0e--- /dev/null+++ b/test/test_cpu/export/test_gguf_mtp_dtype.py@@ -0,0 +1,122 @@+import pytest++from auto_round.export.export_to_gguf.gguf_dtype import (+ GGUFDTypeSelector,+ gguf_format_to_ftype,+ select_llama_cpp_compatible_qtype,+)+from auto_round.utils import LazyImport++gguf = LazyImport("gguf")++[email protected]("layer_id", [0, 28])+def test_attn_v_dtype_uses_same_rule_for_main_and_mtp_layers(layer_id):+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}++ qtype = select_llama_cpp_compatible_qtype(+ f"blk.{layer_id}.attn_v.weight",+ gguf.LlamaFileType.MOSTLY_Q2_K_S,+ hparams,+ n_dims=2,+ )++ assert qtype == gguf.GGMLQuantizationType.Q4_K+++def test_mtp_ffn_down_dtype_follows_llama_cpp_mixed_rule():+ hparams = {"num_hidden_layers": 28, "mtp_num_hidden_layers": 1, "num_attention_heads": 16, "num_key_value_heads": 4}++ qtype = select_llama_cpp_compatible_qtype(+ "blk.28.ffn_down.weight",+ gguf.LlamaFileType.MOSTLY_Q4_K_M,+ hparams,+ n_dims=2,+ )++ assert qtype == gguf.GGMLQuantizationType.Q6_K+++def test_mtp_attn_output_dtype_follows_llama_cpp_mixed_rule():+ hparams = {"num_hidden_layers": 28, "mtp_num_hidden_layers": 1, "num_attention_heads": 16, "num_key_value_heads": 4}++ qtype = select_llama_cpp_compatible_qtype(+ "blk.28.attn_output.weight",+ gguf.LlamaFileType.MOSTLY_Q3_K_M,+ hparams,+ n_dims=2,+ )++ assert qtype == gguf.GGMLQuantizationType.Q4_K+++def test_one_dim_mtp_tensors_stay_f32():+ hparams = {"num_hidden_layers": 28}++ qtype = select_llama_cpp_compatible_qtype(+ "blk.28.ffn_norm.weight",+ gguf.LlamaFileType.MOSTLY_Q4_K_M,+ hparams,+ n_dims=1,+ )++ assert qtype == gguf.GGMLQuantizationType.F32+++def test_selector_tracks_attn_v_counter_for_q3_k_m():+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q3_k_m"))++ qtypes = [selector.select_qtype(f"blk.{idx}.attn_v.weight", n_dims=2) for idx in range(3)]++ assert qtypes == [+ gguf.GGMLQuantizationType.Q5_K,+ gguf.GGMLQuantizationType.Q5_K,+ gguf.GGMLQuantizationType.Q4_K,+ ]+++def test_selector_returns_gguf_type_string_for_compressor_config():+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q2_k_s"))++ assert selector.select_gguf_type("blk.28.attn_v.weight", n_dims=2) == "gguf:q4_k"+++def test_attn_qkv_matches_llama_cpp_attn_v_like_rule():+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q3_k_m"))++ qtype = selector.select_qtype("blk.0.attn_qkv.weight", n_dims=2)++ assert qtype == gguf.GGMLQuantizationType.Q5_K+ assert selector.i_attention_wv == 1+++def test_selector_accepts_explicit_layer_count_from_compressor():+ hparams = {"num_hidden_layers": 2, "num_attention_heads": 16, "num_key_value_heads": 4}+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_k_m"), n_layer=28)++ assert selector.select_gguf_type("blk.1.attn_v.weight", n_dims=2) == "gguf:q6_k"+++def test_selector_uses_cpp_integer_layer_division():+ hparams = {"num_hidden_layers": 2, "num_attention_heads": 16, "num_key_value_heads": 4}+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_k_m"), n_layer=2)++ assert selector.select_gguf_type("blk.1.ffn_down.weight", n_dims=2) == "gguf:q6_k"+++def test_attn_v_more_bits_uses_attention_counter_not_layer_id():+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_k_m"), n_attention_wv=2)++ assert selector.select_gguf_type("blk.0.attn_v.weight", n_dims=2) == "gguf:q4_k"+ assert selector.select_gguf_type("blk.1.attn_v.weight", n_dims=2) == "gguf:q6_k"+++def test_q4_0_ffn_down_upgrade_requires_imatrix_like_llama_cpp():+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_0"), has_imatrix=False)++ assert selector.select_gguf_type("blk.0.ffn_down.weight", n_dims=2) == "gguf:q4_0"diff --git a/test/test_cpu/schemes/test_scheme.py b/test/test_cpu/schemes/test_scheme.pyindex b011d7fc0..3c64be8b9 100644--- a/test/test_cpu/schemes/test_scheme.py+++ b/test/test_cpu/schemes/test_scheme.py@@ -2,10 +2,12 @@ import osimport shutilimport pytest+import torch+import torch.nn as nnimport transformersfrom auto_round import AutoRound-from auto_round.schemes import QuantizationScheme+from auto_round.schemes import QuantizationScheme, _handle_special_schemesfrom ...helpers import get_model_path, get_tiny_model, opt_name_or_path, qwen_name_or_path, save_tiny_model@@ -229,3 +231,86 @@ class TestAutoRound:_, quantized_model_path = ar.quantize_and_save(self.save_folder)model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)assert model is not None, "Model loading failed after quantization with FP8_STATIC scheme"+++class TestQ2KMixedRecipe:+ """Unit tests for the gguf:q2_k_mixed _handle_special_schemes recipe.++ Uses a small dummy model so no real checkpoint is required.+ """++ class _DummyMLP(nn.Module):+ def __init__(self):+ super().__init__()+ self.up_proj = nn.Linear(8, 8)+ self.down_proj = nn.Linear(8, 8)+ self.gate_proj = nn.Linear(8, 8)++ class _DummyMoE(nn.Module):+ def __init__(self):+ super().__init__()+ # singular form used by Qwen2MoE / Qwen3.6 / Qwen3_5MoE+ self.shared_expert = TestQ2KMixedRecipe._DummyMLP()+ self.shared_expert_gate = nn.Linear(8, 1)+ # plural form used by DeepSeek / Mistral-Large+ self.shared_experts = TestQ2KMixedRecipe._DummyMLP()+ # regular routed experts (should NOT be promoted)+ self.experts = nn.ModuleList([nn.Linear(8, 8) for _ in range(2)])+ # fused expert weights stay 3D and should remain q2k in GGUF mapping+ self.gate_up_proj = nn.Linear(8, 16, bias=False)+ self.gate_up_proj.weight = nn.Parameter(torch.zeros(2, 16, 8))+ self.down_proj = nn.Linear(8, 8, bias=False)+ self.down_proj.weight = nn.Parameter(torch.zeros(2, 8, 8))++ class _DummyModel(nn.Module):+ def __init__(self):+ super().__init__()+ self.layer0 = nn.Module()+ self.layer0.mlp = TestQ2KMixedRecipe._DummyMoE()+ # lm_head is Linear but should get Q8_0, not Q4_K_S+ self.lm_head = nn.Linear(8, 8)+ # embedding should get Q8_0+ self.embed_tokens = nn.Embedding(16, 8)++ def setup_method(self):+ self.model = self._DummyModel()+ self.cfg = _handle_special_schemes("gguf:q2_k_mixed", {}, self.model)++ def test_singular_shared_expert_linear_gets_q4ks(self):+ """2D shared expert linears should use Q4_K_S."""+ for sub in ("up_proj", "down_proj", "gate_proj"):+ key = f"layer0.mlp.shared_expert.{sub}"+ assert key in self.cfg, f"{key} missing from layer_config"+ assert self.cfg[key] == "GGUF:Q4_K_S", f"{key} expected Q4_K_S, got {self.cfg[key]}"++ def test_shared_expert_gate_gets_q4ks(self):+ """2D shared expert gate should use Q4_K_S."""+ key = "layer0.mlp.shared_expert_gate"+ assert key in self.cfg, f"{key} missing from layer_config"+ assert self.cfg[key] == "GGUF:Q4_K_S", f"{key} expected Q4_K_S, got {self.cfg[key]}"++ def test_plural_shared_experts_linear_gets_q4ks(self):+ """2D plural shared experts should also use Q4_K_S."""+ for sub in ("up_proj", "down_proj", "gate_proj"):+ key = f"layer0.mlp.shared_experts.{sub}"+ assert key in self.cfg, f"{key} missing from layer_config"+ assert self.cfg[key] == "GGUF:Q4_K_S", f"{key} expected Q4_K_S, got {self.cfg[key]}"++ def test_routed_experts_linears_not_in_recipe(self):+ """Routed expert linears are excluded from the recipe — falls through to dtype selector."""+ for i in range(2):+ key = f"layer0.mlp.experts.{i}"+ assert key not in self.cfg, f"{key} should not be in the recipe"++ def test_fused_three_dim_expert_weights_start_as_q2ks_recipe(self):+ """_handle_special_schemes sets 3D expert weights directly to Q2_K_S."""+ assert self.cfg.get("layer0.mlp.gate_up_proj") == "GGUF:Q2_K_S"+ assert self.cfg.get("layer0.mlp.down_proj") == "GGUF:Q2_K_S"++ def test_lm_head_gets_q8_0(self):+ """lm_head should be Q8_0."""+ assert self.cfg.get("lm_head") == "GGUF:Q8_0"++ def test_embedding_gets_q8_0(self):+ """embed_tokens embedding should be Q8_0."""+ assert self.cfg.get("embed_tokens") == "GGUF:Q8_0"diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.pyindex fb4069048..458266b86 100644--- a/test/test_cuda/export/test_gguf_format.py+++ b/test/test_cuda/export/test_gguf_format.py@@ -177,7 +177,7 @@ class TestAutoRound:if "mmproj-model.gguf" in file:assert abs(file_size - 75) < 5.0else:- assert abs(file_size - 683) < 5.0+ assert abs(file_size - 690) < 10.0shutil.rmtree(tiny_model_path, ignore_errors=True)