Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

gguf format add support for mtp quantization (#1866)

Heng Guof7e77e034d4f
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index a8a7bc5ab..f970d89f9 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -554,8 +554,9 @@ class BaseCompressor(object):
is_gguf_format = _formats is not None and any(
"gguf" in str(getattr(fmt, "output_format", "")) for fmt in _formats
)
- predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)
+ predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model) if not is_gguf_format else []
compressed_predefined_ignore_layers = compress_layer_names(predefined_ignore_layers)
+
if not is_gguf_format:
predefined_ignore_layers = get_predefined_ignore_layers(self.model_context.model)
if predefined_ignore_layers and self.quant_block_list:
@@ -1063,7 +1064,15 @@ class BaseCompressor(object):
self._offloader.reset()
# Disable inplace when quantized layers live outside transformer blocks.
- if self.has_qlayer_outside_block and self.need_calib:
+ # gguf lm-head used rtn in version>=0.13
+ if (
+ self.has_qlayer_outside_block
+ and self.need_calib
+ and (
+ self.compress_context.formats is None
+ or "gguf" not in self.compress_context.formats[0].__class__.__name__.lower()
+ )
+ ):
self.inplace = False
if not hasattr(self, "formats"):
@@ -1222,12 +1231,14 @@ class BaseCompressor(object):
return
formats = getattr(self, "formats", [])
- if len(formats) == 1 and not formats[0].is_fake() and self.inplace:
+ has_single_gguf_format = len(formats) == 1 and formats[0].is_gguf()
+ # GGUF supports per-block / per-layer immediate packing even when
+ # full-model in-place rewriting is disabled by outside-block layers.
+ if len(formats) == 1 and not formats[0].is_fake() and (self.inplace or has_single_gguf_format):
self.compress_context.is_immediate_packing = True
- if self.has_qlayer_outside_block and self.need_calib:
+ if self.has_qlayer_outside_block and self.need_calib and not has_single_gguf_format:
self.compress_context.is_immediate_packing = False
-
if not ("causallm" in self.model_context.model.__class__.__name__.lower() and not self.model_context.is_mllm):
# TODO For tied keys, there may some issues, we haven't not verified this
tied_weight_keys = getattr(self.model_context.model, "_tied_weight_keys", {})
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 1ef4e5720..3050b62eb 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -26,6 +26,7 @@ import transformers
from torch.amp import autocast
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, GGUF_CONFIG, GGUF_INNER_CONFIG, QK_K, ModelType
+from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector, gguf_format_to_ftype
from auto_round.logger import logger
from auto_round.utils import (
check_to_quantized,
@@ -774,25 +775,36 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model
tie_word_embeddings = model.config.tie_word_embeddings
tie_word_embeddings &= not is_separate_lm_head(model)
- n_gqa = 1
- if (
- hasattr(model, "config")
- and hasattr(model.config, "num_attention_heads")
- and hasattr(model.config, "num_key_value_heads")
- ):
- n_gqa = model.config.num_attention_heads // model.config.num_key_value_heads
- n_expert = 0
- for name in ["num_experts", "num_local_experts", "n_routed_experts"]:
- if hasattr(model.config, name):
- n_expert = getattr(model.config, name)
-
- i_attention_wv = 0
- i_ffn_down = 0
+ dtype_selector = GGUFDTypeSelector(
+ hparams, gguf_format_to_ftype(target_gguf_format), model_class.model_arch, n_layer
+ )
layer_config_copy = copy.deepcopy(layer_config)
base_target_bits = None
if inner_gguf_format.startswith("gguf:q") and len(inner_gguf_format) >= 7 and (inner_gguf_format[6]).isdigit():
base_target_bits = int(inner_gguf_format[6])
+ def _resolve_gguf_name(layer_name):
+ if model_type != ModelType.TEXT and any([key in layer_name for key in MM_MODULE_KEYS]):
+ gguf_layer_name = tensor_map_vision.get_name(layer_name)
+ if gguf_layer_name is None:
+ for key in MM_MODULE_KEYS:
+ gguf_layer_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))
+ if gguf_layer_name is not None:
+ break
+ else:
+ gguf_layer_name = tensor_map.get_name(layer_name)
+ if gguf_layer_name is None:
+ gguf_layer_name = tensor_map.get_name(layer_name.replace(".language_model", ""))
+ return gguf_layer_name
+
+ dtype_selector.n_attention_wv = sum(
+ 1
+ for layer_name, config in layer_config_copy.items()
+ if check_to_quantized(config)
+ and (gguf_name := _resolve_gguf_name(layer_name))
+ and any(key in gguf_name for key in ("attn_v", "attn_qkv", "attn_kv_b"))
+ )
+
for layer_name, config in layer_config_copy.items():
if not check_to_quantized(config):
continue
@@ -815,17 +827,7 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model
re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format][embedding_format_key]).group(1)
)
- if model_type != ModelType.TEXT and any([key in layer_name for key in MM_MODULE_KEYS]):
- gguf_name = tensor_map_vision.get_name(layer_name)
- if gguf_name is None:
- for key in MM_MODULE_KEYS:
- gguf_name = tensor_map_vision.get_name(layer_name.replace(f".{key}", ""))
- if gguf_name is not None:
- break
- else:
- gguf_name = tensor_map.get_name(layer_name)
- if gguf_name is None:
- gguf_name = tensor_map.get_name(layer_name.replace(".language_model", ""))
+ gguf_name = _resolve_gguf_name(layer_name)
bits_index = 6
if config.get("fixed_by_user", False):
if "bits" not in config:
@@ -877,116 +879,9 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model
new_type = _search_gguf_type(new_type)
if new_type is None:
raise ValueError(f"invalid bit setting for {layer_name}")
- elif gguf_name is None:
- pass
- # attn_v
- elif "attn_v" in gguf_name:
- if target_gguf_format == "gguf:q2_k":
- new_type = "gguf:q4_k" if n_gqa >= 4 else "gguf:q3_k"
- elif target_gguf_format == "gguf:q2_k_s" and n_gqa >= 4:
- new_type = "gguf:q4_k"
- elif target_gguf_format == "gguf:q3_k_m":
- new_type = "gguf:q5_k" if i_attention_wv < 2 else "gguf:q4_k"
- elif target_gguf_format == "gguf:q3_k_l":
- new_type = "gguf:q5_k"
- elif (target_gguf_format == "gguf:q4_k_m" or target_gguf_format == "gguf:q5_k_m") and _use_more_bits(
- i_layer, n_layer
- ):
- new_type = "gguf:q6_k"
- elif target_gguf_format == "gguf:q4_k_s" and i_attention_wv < 4:
- new_type = "gguf:q5_k"
- ##TODO check which models are be grouped into to LLM_TYPE_70B
- # if (qs.model.type == LLM_TYPE_70B) {
- # // In the 70B model we have 8 heads sharing the same attn_v weights.
- # As a result, the attn_v.weight tensor is
- # // 8x smaller compared to attn_q.weight.Hence, we can get a nice boost in quantization accuracy with
- # // nearly negligible increase in model size by quantizing this tensor with more bits:
- # if
- # (new_type == GGML_TYPE_Q3_K | | new_type == GGML_TYPE_Q4_K)
- # new_type = GGML_TYPE_Q5_K;
- # }
- if n_expert == 8:
- new_type = "gguf:q8_k"
- i_attention_wv += 1
-
- elif "attn_k" in gguf_name:
- if n_expert == 8:
- new_type = "gguf:q8_0"
- # ffn_down
- elif "ffn_down" in gguf_name:
- if target_gguf_format == "gguf:q2_k":
- new_type = "gguf:q3_k"
- elif target_gguf_format == "gguf:q2_k_s":
- if i_layer < n_layer / 8:
- new_type = "gguf:q4_k"
- elif target_gguf_format == "gguf:q3_k_m":
- if i_layer < n_layer / 16:
- new_type = "gguf:q5_k"
- elif gguf.MODEL_ARCH.FALCON == model_class.model_arch or _use_more_bits(i_layer, n_layer):
- new_type = "gguf:q4_k"
- else:
- new_type = "gguf:q3_k"
- elif target_gguf_format == "gguf:q3_k_l":
- if gguf.MODEL_ARCH.FALCON == model_class.model_arch:
- new_type = "gguf:q4_k"
- else:
- new_type = "gguf:q5_k"
- elif target_gguf_format == "gguf:q4_k_m":
- if gguf.MODEL_ARCH.FALCON == model_class.model_arch:
- if i_layer < n_layer // 16:
- new_type = "gguf:q6_k"
- elif _use_more_bits(i_layer, n_layer):
- new_type = "gguf:q5_k"
- else:
- new_type = "gguf:q4_k"
- else:
- if _use_more_bits(i_layer, n_layer):
- new_type = "gguf:q6_k"
- elif target_gguf_format == "gguf:q5_k_m" and _use_more_bits(i_layer, n_layer):
- new_type = "gguf:q6_k"
- elif (
- target_gguf_format == "gguf:q4_k_s"
- and model_class.model_arch != gguf.MODEL_ARCH.FALCON
- and i_layer < n_layer / 8
- ):
- new_type = "gguf:q5_k"
- elif (target_gguf_format == "gguf:q4_0" or target_gguf_format == "gguf:q5_0") and i_layer < n_layer / 8:
- if target_gguf_format == "gguf:q4_0":
- new_type = "gguf:q4_1"
- else:
- new_type = "gguf:q5_1"
- i_ffn_down += 1
-
- # attn_output
- elif "attn_output" in gguf_name:
- if gguf.MODEL_ARCH.FALCON != model_class.model_arch:
- if n_expert == 8:
- if target_gguf_format in (
- "gguf:q2_k",
- "gguf:q3_k_s",
- "gguf:q3_k_m",
- "gguf:q4_k_s",
- "gguf:q4_k_m",
- "gguf:q5_k",
- ):
- new_type = "gguf:q5_k"
- elif target_gguf_format == "gguf:q2_k":
- new_type = "gguf:q3_k"
- elif target_gguf_format == "gguf:q3_k_m":
- new_type = "gguf:q4_k"
- elif target_gguf_format == "gguf:q3_k_l":
- new_type = "gguf:q5_k"
- else:
- if target_gguf_format == "gguf:q3_k_l":
- new_type = "gguf:q4_k"
- # attn_qkv
- elif "attn_qkv" in gguf_name:
- if target_gguf_format in ("gguf:q3_k_m", "gguf:q3_k_l"):
- new_type = "gguf:q4_k"
- elif target_gguf_format == "gguf:q4_k_m":
- new_type = "gguf:q5_k"
- elif target_gguf_format == "gguf:q5_k_m":
- new_type = "gguf:q5_k"
+ elif gguf_name is not None:
+ gguf_weight_name = gguf_name if gguf_name.endswith(".weight") else f"{gguf_name}.weight"
+ new_type = dtype_selector.select_gguf_type(gguf_weight_name, len(layer.weight.shape), i_layer or 0)
new_block_size = GGML_QUANT_SIZES[new_type.split(":")[-1].lower()][0]
if input_features % new_block_size != 0:
new_type = _gguf_type_fallback(new_type)
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 0af48ce61..539c90c00 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -47,6 +47,7 @@ import torch
from transformers import AutoConfig
from auto_round.export.export_to_gguf.config import ModelType
+from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector
from auto_round.export.export_to_gguf.packing import ggml_quant
from auto_round.utils import (
LazyImport,
@@ -297,7 +298,7 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d
return data, data_qtype
-def get_qtype_by_layer_config(layer_config, name, data_qtype):
+def get_qtype_by_layer_config(layer_config, name, data_qtype, *, explicit_only=False):
name = name[: -len(".weight")]
if name not in layer_config and name.endswith("embed_tokens"):
embedding_names = [key for key in layer_config if key.endswith("embed_tokens")]
@@ -307,7 +308,9 @@ def get_qtype_by_layer_config(layer_config, name, data_qtype):
embedding_names = [key for key in layer_config if key.endswith("embed_tokens")]
if len(embedding_names) == 1:
name = embedding_names[0]
- if name not in layer_config or layer_config[name]["bits"] >= 16:
+ if name not in layer_config:
+ return None if explicit_only else data_qtype
+ if layer_config[name]["bits"] >= 16:
return data_qtype
bits = layer_config[name].get("bits")
super_bits = layer_config[name].get("super_bits")
@@ -413,8 +416,28 @@ def _special_name_handle(cls, name):
return name
+def _count_attention_wv_tensors(cls):
+ count = 0
+ for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):
+ if data_torch is None or data_torch.numel() == 0:
+ continue
+ filtered = cls.filter_tensors((name, lambda data_torch=data_torch: data_torch))
+ if filtered is None:
+ continue
+ modify_name = _special_name_handle(cls, filtered[0])
+ try:
+ new_name = cls.map_tensor_name(modify_name)
+ except Exception:
+ continue
+ count += any(key in new_name for key in ("attn_v.weight", "attn_qkv.weight", "attn_kv_b.weight"))
+ return count or None
+
+
def prepare_tensors(cls):
device = get_packing_device(cls.device)
+ if not hasattr(cls, "_gguf_dtype_selector"):
+ cls._gguf_dtype_selector = GGUFDTypeSelector(cls.hparams, cls.ftype, cls.model_arch)
+ cls._gguf_dtype_selector.n_attention_wv = _count_attention_wv_tensors(cls)
# Handle empty tensor_map for models with block_count=0 (like MobileNetV5)
if cls.tensor_map.mapping:
@@ -463,7 +486,6 @@ def prepare_tensors(cls):
clean_weight_list = []
modify_name = _special_name_handle(cls, name)
- data_torch = data_torch.to("cpu")
for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):
skip = False
for tensor_info in cls.gguf_writer.tensors:
@@ -539,8 +561,13 @@ def prepare_tensors(cls):
name = get_moe_name(cls, name, new_name)
clean_weight_list.append(name)
# get data_qtype by layer_config
- data_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype)
+ layer_config_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype, explicit_only=True)
# # No override (data_qtype is False), or wants to be quantized (data_qtype is True)
+ if layer_config_qtype is not None:
+ data_qtype = layer_config_qtype
+ elif isinstance(data_qtype, bool):
+ data_qtype = cls._gguf_dtype_selector.select_qtype(new_name, n_dims)
+
if isinstance(data_qtype, bool):
if cls.ftype == gguf.LlamaFileType.ALL_F32:
data_qtype = gguf.GGMLQuantizationType.F32
@@ -611,7 +638,7 @@ def prepare_tensors(cls):
# if data ends up empty, it means data_torch was a scalar tensor -> restore
if len(data_torch.shape) == 0:
- data = data_torch.numpy()
+ data = data_torch.cpu().numpy()
try:
data = gguf.quants.quantize(data, data_qtype)
except gguf.QuantError as e:
diff --git a/auto_round/export/export_to_gguf/gguf_dtype.py b/auto_round/export/export_to_gguf/gguf_dtype.py
new file mode 100644
index 000000000..93c9f2ff9
--- /dev/null
+++ b/auto_round/export/export_to_gguf/gguf_dtype.py
@@ -0,0 +1,320 @@
+# Copyright (c) 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from enum import Enum
+
+from auto_round.utils import LazyImport
+
+gguf = LazyImport("gguf")
+
+
+_GGUF_TYPE_TO_QTYPE_NAME = {
+ "gguf:fp16": "F16",
+ "gguf:f16": "F16",
+ "gguf:bf16": "BF16",
+ "gguf:f32": "F32",
+ "gguf:q2_k": "Q2_K",
+ "gguf:q3_k": "Q3_K",
+ "gguf:q4_k": "Q4_K",
+ "gguf:q5_k": "Q5_K",
+ "gguf:q6_k": "Q6_K",
+ "gguf:q4_0": "Q4_0",
+ "gguf:q4_1": "Q4_1",
+ "gguf:q5_0": "Q5_0",
+ "gguf:q5_1": "Q5_1",
+ "gguf:q8_0": "Q8_0",
+}
+
+
+_QTYPE_NAME_TO_GGUF_TYPE = {qtype: gguf_type for gguf_type, qtype in _GGUF_TYPE_TO_QTYPE_NAME.items()}
+_QTYPE_NAME_TO_GGUF_TYPE["F16"] = "gguf:fp16"
+
+
+class TensorCategory(Enum):
+ TOKEN_EMBD = "token_embd"
+ ATTENTION_Q = "attn_q"
+ ATTENTION_V = "attn_v"
+ ATTENTION_K = "attn_k"
+ ATTENTION_QKV = "attn_qkv"
+ ATTENTION_KV_B = "attn_kv_b"
+ ATTENTION_OUTPUT = "attn_output"
+ FFN_UP = "ffn_up"
+ FFN_GATE = "ffn_gate"
+ FFN_DOWN = "ffn_down"
+ OUTPUT = "output"
+ OTHER = "other"
+
+
+def _tensor_category(name: str) -> TensorCategory:
+ if name == "output.weight":
+ return TensorCategory.OUTPUT
+ if name in ("token_embd.weight", "per_layer_token_embd.weight"):
+ return TensorCategory.TOKEN_EMBD
+ if "attn_qkv.weight" in name:
+ return TensorCategory.ATTENTION_QKV
+ if "attn_kv_b.weight" in name:
+ return TensorCategory.ATTENTION_KV_B
+ if "attn_v.weight" in name:
+ return TensorCategory.ATTENTION_V
+ if "attn_k.weight" in name:
+ return TensorCategory.ATTENTION_K
+ if "attn_q.weight" in name:
+ return TensorCategory.ATTENTION_Q
+ if "attn_output.weight" in name:
+ return TensorCategory.ATTENTION_OUTPUT
+ if "ffn_up" in name:
+ return TensorCategory.FFN_UP
+ if "ffn_gate" in name:
+ return TensorCategory.FFN_GATE
+ if "ffn_down" in name:
+ return TensorCategory.FFN_DOWN
+ return TensorCategory.OTHER
+
+
+def _is_attn_v_like(category: TensorCategory) -> bool:
+ return category in (TensorCategory.ATTENTION_V, TensorCategory.ATTENTION_QKV, TensorCategory.ATTENTION_KV_B)
+
+
+def _use_more_bits(i_layer: int, n_layer: int) -> bool:
+ return i_layer < n_layer // 8 or i_layer >= 7 * n_layer // 8 or (i_layer - n_layer // 8) % 3 == 2
+
+
+def _get_layer_id(name: str, fallback: int) -> int:
+ parts = name.split(".")
+ if len(parts) > 1 and parts[0] == "blk" and parts[1].isdigit():
+ return int(parts[1])
+ return fallback
+
+
+def _default_qtype(ftype):
+ if ftype == gguf.LlamaFileType.ALL_F32:
+ return gguf.GGMLQuantizationType.F32
+ if ftype == gguf.LlamaFileType.MOSTLY_F16:
+ return gguf.GGMLQuantizationType.F16
+ if ftype == gguf.LlamaFileType.MOSTLY_BF16:
+ return gguf.GGMLQuantizationType.BF16
+ if ftype == gguf.LlamaFileType.MOSTLY_Q8_0:
+ return gguf.GGMLQuantizationType.Q8_0
+ if ftype == gguf.LlamaFileType.MOSTLY_Q4_0:
+ return gguf.GGMLQuantizationType.Q4_0
+ if ftype == gguf.LlamaFileType.MOSTLY_Q4_1:
+ return gguf.GGMLQuantizationType.Q4_1
+ if ftype == gguf.LlamaFileType.MOSTLY_Q5_0:
+ return gguf.GGMLQuantizationType.Q5_0
+ if ftype == gguf.LlamaFileType.MOSTLY_Q5_1:
+ return gguf.GGMLQuantizationType.Q5_1
+ if ftype in (gguf.LlamaFileType.MOSTLY_Q2_K_S, gguf.LlamaFileType.MOSTLY_Q2_K):
+ return gguf.GGMLQuantizationType.Q2_K
+ if ftype in (
+ gguf.LlamaFileType.MOSTLY_Q3_K_S,
+ gguf.LlamaFileType.MOSTLY_Q3_K_M,
+ gguf.LlamaFileType.MOSTLY_Q3_K_L,
+ ):
+ return gguf.GGMLQuantizationType.Q3_K
+ if ftype in (gguf.LlamaFileType.MOSTLY_Q4_K_S, gguf.LlamaFileType.MOSTLY_Q4_K_M):
+ return gguf.GGMLQuantizationType.Q4_K
+ if ftype in (gguf.LlamaFileType.MOSTLY_Q5_K_S, gguf.LlamaFileType.MOSTLY_Q5_K_M):
+ return gguf.GGMLQuantizationType.Q5_K
+ if ftype == gguf.LlamaFileType.MOSTLY_Q6_K:
+ return gguf.GGMLQuantizationType.Q6_K
+ raise ValueError(f"Unknown file type: {ftype.name}")
+
+
+def gguf_format_to_ftype(gguf_format: str):
+ format_name = gguf_format.lower()
+ if format_name == "gguf:q2_k_mixed":
+ format_name = "gguf:q2_k_s"
+ mapping = {
+ "gguf:f32": gguf.LlamaFileType.ALL_F32,
+ "gguf:fp16": gguf.LlamaFileType.MOSTLY_F16,
+ "gguf:f16": gguf.LlamaFileType.MOSTLY_F16,
+ "gguf:bf16": gguf.LlamaFileType.MOSTLY_BF16,
+ "gguf:q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,
+ "gguf:q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,
+ "gguf:q5_0": gguf.LlamaFileType.MOSTLY_Q5_0,
+ "gguf:q5_1": gguf.LlamaFileType.MOSTLY_Q5_1,
+ "gguf:q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,
+ "gguf:q2_k": gguf.LlamaFileType.MOSTLY_Q2_K,
+ "gguf:q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,
+ "gguf:q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,
+ "gguf:q3_k_m": gguf.LlamaFileType.MOSTLY_Q3_K_M,
+ "gguf:q3_k_l": gguf.LlamaFileType.MOSTLY_Q3_K_L,
+ "gguf:q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,
+ "gguf:q4_k_m": gguf.LlamaFileType.MOSTLY_Q4_K_M,
+ "gguf:q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,
+ "gguf:q5_k_m": gguf.LlamaFileType.MOSTLY_Q5_K_M,
+ "gguf:q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,
+ "gguf:q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,
+ }
+ if format_name not in mapping:
+ raise ValueError(f"Unknown GGUF format: {gguf_format}")
+ return mapping[format_name]
+
+
+def qtype_to_gguf_type(qtype) -> str:
+ return _QTYPE_NAME_TO_GGUF_TYPE[qtype.name]
+
+
+class GGUFDTypeSelector:
+ def __init__(
+ self,
+ hparams: dict,
+ ftype,
+ model_arch=None,
+ n_layer: int | None = None,
+ n_attention_wv: int | None = None,
+ has_imatrix: bool = False,
+ ):
+ self.hparams = hparams
+ self.ftype = ftype
+ self.model_arch = model_arch
+ self.n_layer = n_layer
+ self.n_attention_wv = n_attention_wv
+ self.has_imatrix = has_imatrix
+ self.i_attention_wv = 0
+ self.i_ffn_down = 0
+
+ def select_qtype(self, name: str, n_dims: int, fallback_index: int = 0):
+ """Select the GGML tensor dtype used by llama.cpp quantization.
+
+ `name` must be the final GGUF tensor name. MTP/NextN tensors are already
+ remapped to extended `blk.<id>` names by conversion classes, so they use
+ the same category and layer-index rules as regular transformer blocks.
+ """
+ if n_dims < 2:
+ return gguf.GGMLQuantizationType.F32
+
+ qtype = _default_qtype(self.ftype)
+ category = _tensor_category(name)
+ i_layer = _get_layer_id(name, fallback_index)
+ if self.n_layer is None:
+ n_layer = int(
+ self.hparams.get("num_hidden_layers", self.hparams.get("n_layer", self.hparams.get("num_layers", 1)))
+ )
+ n_layer += int(
+ self.hparams.get("mtp_num_hidden_layers", self.hparams.get("num_nextn_predict_layers", 0)) or 0
+ )
+ else:
+ n_layer = self.n_layer
+ n_layer = max(n_layer, i_layer + 1)
+ n_gqa = 1
+ n_head = self.hparams.get("num_attention_heads", self.hparams.get("n_head"))
+ n_head_kv = self.hparams.get("num_key_value_heads", self.hparams.get("n_head_kv"))
+ if n_head and n_head_kv:
+ n_gqa = max(1, int(n_head) // int(n_head_kv))
+ n_expert = int(
+ self.hparams.get(
+ "num_experts", self.hparams.get("num_local_experts", self.hparams.get("n_routed_experts", 0))
+ )
+ or 0
+ )
+
+ if _is_attn_v_like(category):
+ if self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:
+ qtype = gguf.GGMLQuantizationType.Q4_K if n_gqa >= 4 else gguf.GGMLQuantizationType.Q3_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S and n_gqa >= 4:
+ qtype = gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M:
+ qtype = gguf.GGMLQuantizationType.Q5_K if self.i_attention_wv < 2 else gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:
+ qtype = gguf.GGMLQuantizationType.Q5_K
+ elif self.ftype in (gguf.LlamaFileType.MOSTLY_Q4_K_M, gguf.LlamaFileType.MOSTLY_Q5_K_M) and _use_more_bits(
+ self.i_attention_wv, self.n_attention_wv or n_layer
+ ):
+ qtype = gguf.GGMLQuantizationType.Q6_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S and self.i_attention_wv < 4:
+ qtype = gguf.GGMLQuantizationType.Q5_K
+ if n_expert == 8:
+ qtype = gguf.GGMLQuantizationType.Q8_0
+ self.i_attention_wv += 1
+ elif category == TensorCategory.ATTENTION_K and n_expert == 8:
+ qtype = gguf.GGMLQuantizationType.Q8_0
+ elif category == TensorCategory.FFN_DOWN:
+ if self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:
+ qtype = gguf.GGMLQuantizationType.Q3_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S and i_layer < n_layer // 8:
+ qtype = gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M:
+ if i_layer < n_layer // 16:
+ qtype = gguf.GGMLQuantizationType.Q5_K
+ elif self.model_arch == gguf.MODEL_ARCH.FALCON or _use_more_bits(i_layer, n_layer):
+ qtype = gguf.GGMLQuantizationType.Q4_K
+ else:
+ qtype = gguf.GGMLQuantizationType.Q3_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:
+ qtype = (
+ gguf.GGMLQuantizationType.Q4_K
+ if self.model_arch == gguf.MODEL_ARCH.FALCON
+ else gguf.GGMLQuantizationType.Q5_K
+ )
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:
+ if self.model_arch == gguf.MODEL_ARCH.FALCON:
+ if i_layer < n_layer // 16:
+ qtype = gguf.GGMLQuantizationType.Q6_K
+ elif _use_more_bits(i_layer, n_layer):
+ qtype = gguf.GGMLQuantizationType.Q5_K
+ elif _use_more_bits(i_layer, n_layer):
+ qtype = gguf.GGMLQuantizationType.Q6_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M and _use_more_bits(i_layer, n_layer):
+ qtype = gguf.GGMLQuantizationType.Q6_K
+ elif (
+ self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S
+ and self.model_arch != gguf.MODEL_ARCH.FALCON
+ and i_layer < n_layer // 8
+ ):
+ qtype = gguf.GGMLQuantizationType.Q5_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0 and self.has_imatrix and i_layer < n_layer // 8:
+ qtype = gguf.GGMLQuantizationType.Q4_1
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0 and self.has_imatrix and i_layer < n_layer // 8:
+ qtype = gguf.GGMLQuantizationType.Q5_1
+ self.i_ffn_down += 1
+ elif category == TensorCategory.ATTENTION_OUTPUT:
+ if self.model_arch != gguf.MODEL_ARCH.FALCON:
+ if n_expert == 8:
+ if self.ftype in (
+ gguf.LlamaFileType.MOSTLY_Q2_K,
+ gguf.LlamaFileType.MOSTLY_Q3_K_S,
+ gguf.LlamaFileType.MOSTLY_Q3_K_M,
+ gguf.LlamaFileType.MOSTLY_Q4_K_S,
+ gguf.LlamaFileType.MOSTLY_Q4_K_M,
+ ):
+ qtype = gguf.GGMLQuantizationType.Q5_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:
+ qtype = gguf.GGMLQuantizationType.Q3_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M:
+ qtype = gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:
+ qtype = gguf.GGMLQuantizationType.Q5_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_L:
+ qtype = gguf.GGMLQuantizationType.Q4_K
+ elif category == TensorCategory.ATTENTION_QKV:
+ if self.ftype in (gguf.LlamaFileType.MOSTLY_Q3_K_M, gguf.LlamaFileType.MOSTLY_Q3_K_L):
+ qtype = gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:
+ qtype = gguf.GGMLQuantizationType.Q5_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_M:
+ qtype = gguf.GGMLQuantizationType.Q6_K
+
+ return qtype
+
+ def select_gguf_type(self, name: str, n_dims: int, fallback_index: int = 0) -> str:
+ return qtype_to_gguf_type(self.select_qtype(name, n_dims, fallback_index=fallback_index))
+
+
+def select_llama_cpp_compatible_qtype(name: str, ftype, hparams: dict, n_dims: int, fallback_index: int = 0):
+ selector = GGUFDTypeSelector(hparams, ftype)
+ selector.i_attention_wv = fallback_index
+ return selector.select_qtype(name, n_dims, fallback_index=fallback_index)
diff --git a/auto_round/formats.py b/auto_round/formats.py
index 80313e824..57b6e4f3c 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -780,9 +780,19 @@ class GGUFFormat(OutputFormat):
gguf_format = f"gguf:{format.lower()}"
if format.lower().endswith("_mixed"):
from auto_round.schemes import _handle_special_schemes
+ from auto_round.utils.model import is_moe_model
- ar.layer_config = _handle_special_schemes(gguf_format, ar.layer_config, ar.model)
- gguf_format = gguf_format.lower().replace("_mixed", "_s")
+ if format.lower() == "q2_k_mixed" and getattr(ar, "iters", 0) > 0 and not is_moe_model(ar.model):
+ logger.warning(
+ "gguf:q2_k_mixed only supports MoE models with iters>0. "
+ "It is not an MoE model, falling back to gguf:q4_k_m."
+ )
+ gguf_format = "gguf:q4_k_m"
+ else:
+ ar.layer_config = _handle_special_schemes(
+ gguf_format, ar.layer_config, ar.model, quant_nontext_module=ar.quant_nontext_module
+ )
+ gguf_format = gguf_format.lower().replace("_mixed", "_s")
if isinstance(scheme, str) and scheme.lower() != gguf_format:
logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")
ar.scheme = gguf_format
diff --git a/auto_round/schemes.py b/auto_round/schemes.py
index a298575aa..d351a3f07 100644
--- a/auto_round/schemes.py
+++ b/auto_round/schemes.py
@@ -19,7 +19,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union
import torch
from auto_round.logger import logger
-from auto_round.utils import SUPPORTED_DTYPES, infer_bits_by_data_type
+from auto_round.utils import SUPPORTED_DTYPES, contain_any_mm_keys, infer_bits_by_data_type
__all__ = ["QuantizationScheme", "get_gguf_scheme", "preset_name_to_scheme"]
@@ -576,6 +576,7 @@ def _handle_special_schemes(
inner_supported_types=None,
quant_lm_head=False,
mllm=False,
+ quant_nontext_module=False,
) -> dict:
"""handle special schemes, like GGUF:Q2_K_MIXED.
Provide some special auto_round recipes.
@@ -589,10 +590,16 @@ def _handle_special_schemes(
for n, m in model.named_modules():
if n in layer_config:
continue
+ if isinstance(m, torch.nn.Linear) and contain_any_mm_keys(n) and not quant_nontext_module:
+ layer_config[n] = "BF16"
+ continue
if n == "lm_head" or isinstance(m, torch.nn.Embedding):
layer_config[n] = "GGUF:Q8_0"
- elif isinstance(m, torch.nn.Linear) and ("expert" not in n or "shared_experts" in n) and n != "lm_head":
- layer_config[n] = "GGUF:Q4_K_S"
+ elif isinstance(m, torch.nn.Linear) and ("expert" not in n or "shared_expert" in n) and n != "lm_head":
+ if m.weight.ndim == 3:
+ layer_config[n] = "GGUF:Q2_K_S"
+ else:
+ layer_config[n] = "GGUF:Q4_K_S"
if scheme_name.lower() == "w4a16_mixed":
logger.warning("W4A16_MIXED is experimental and the recipe may change in the future.")
from auto_round.utils import get_lm_head_name
@@ -609,6 +616,13 @@ def _handle_special_schemes(
for n, m in model.named_modules():
if n in layer_config:
continue
+ if (
+ (type(m) in supported_types or type(m) in inner_supported_types)
+ and contain_any_mm_keys(n)
+ and quant_nontext_module
+ ):
+ layer_config[n] = {"bits": 16}
+ continue
if type(m) in supported_types or type(m) in inner_supported_types:
if "expert" in n and "shared" not in n:
layer_config[n] = {"bits": 4, "data_type": "int"}
diff --git a/auto_round/utils/common.py b/auto_round/utils/common.py
index c8b7691cf..5e7a50e6a 100644
--- a/auto_round/utils/common.py
+++ b/auto_round/utils/common.py
@@ -688,6 +688,13 @@ MM_MODULE_KEYS = [
MM_KEYS = [*MM_MODULE_KEYS, "speech", "wav", "waveform"]
+def contain_any_mm_keys(name: str) -> bool:
+ for key in MM_MODULE_KEYS:
+ if key in name:
+ return True
+ return False
+
+
def is_debug_mode():
"""Checks if the Python interpreter is running in debug mode.
diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.py
index cb76df646..56daf87f9 100644
--- a/test/test_cpu/export/test_gguf_format.py
+++ b/test/test_cpu/export/test_gguf_format.py
@@ -142,7 +142,7 @@ class TestGGUF:
assert False, "cmd line test fail, please have a check"
shutil.rmtree("../../tmp_autoround", ignore_errors=True)
- # test mixed q2_k_s
+ # test q2_k_mixed with iters=0 (RTN) on non-MoE model — should still work
res = os.system(
f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {model_name}"
f" --bs 16 --iters 0 --disable_opt_rtn --nsamples 1 --seqlen 16 --scheme GGUF:Q2_K_MIXED"
@@ -151,6 +151,15 @@ class TestGGUF:
assert False, "cmd line test fail, please have a check"
shutil.rmtree("../../tmp_autoround", ignore_errors=True)
+ # test q2_k_mixed with iters=1 on non-MoE model — should fallback to q4_k_m
+ res = os.system(
+ f"PYTHONPATH='{AUTO_ROUND_PATH}:$PYTHONPATH' {python_path} -m auto_round --model {model_name}"
+ f" --bs 16 --iters 1 --nsamples 1 --seqlen 16 --format gguf:q2_k_mixed"
+ )
+ if res > 0 or res == -1:
+ assert False, "cmd line test fail, please have a check"
+ shutil.rmtree("../../tmp_autoround", ignore_errors=True)
+
def test_vlm_gguf(self, tiny_qwen_vl_model_path):
from auto_round import AutoRound
@@ -313,3 +322,25 @@ class TestGGUF:
assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"
assert gguf_model.get_tensor(9).name == "blk.0.ffn_up_exps.weight"
assert gguf_model.get_tensor(9).tensor_type.name == "Q2_K"
+
+ def test_q2k_mixed_keeps_only_three_dim_expert_weights_at_q2k(self, tiny_qwen_moe_model_path):
+ model_name = tiny_qwen_moe_model_path
+ autoround = AutoRound(
+ model_name,
+ iters=0,
+ nsamples=1,
+ seqlen=16,
+ disable_opt_rtn=True,
+ )
+ _, quantized_model_path = autoround.quantize_and_save(output_dir=self.save_dir, format="gguf:q2_k_mixed")
+ gguf_file = os.listdir(quantized_model_path)[0]
+ from gguf.gguf_reader import GGUFReader
+
+ gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))
+ tensor_types = {tensor.name: tensor.tensor_type.name for tensor in gguf_model.tensors}
+
+ assert tensor_types["token_embd.weight"] == "Q8_0"
+ assert tensor_types["output.weight"] == "Q8_0"
+ assert tensor_types["blk.0.attn_k.weight"] == "Q4_K"
+ assert tensor_types["blk.0.ffn_down_exps.weight"] == "Q5_0"
+ assert tensor_types["blk.0.ffn_up_exps.weight"] == "Q2_K"
diff --git a/test/test_cpu/export/test_gguf_mtp_dtype.py b/test/test_cpu/export/test_gguf_mtp_dtype.py
new file mode 100644
index 000000000..a036b8a0e
--- /dev/null
+++ b/test/test_cpu/export/test_gguf_mtp_dtype.py
@@ -0,0 +1,122 @@
+import pytest
+
+from auto_round.export.export_to_gguf.gguf_dtype import (
+ GGUFDTypeSelector,
+ gguf_format_to_ftype,
+ select_llama_cpp_compatible_qtype,
+)
+from auto_round.utils import LazyImport
+
+gguf = LazyImport("gguf")
+
+
[email protected]("layer_id", [0, 28])
+def test_attn_v_dtype_uses_same_rule_for_main_and_mtp_layers(layer_id):
+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}
+
+ qtype = select_llama_cpp_compatible_qtype(
+ f"blk.{layer_id}.attn_v.weight",
+ gguf.LlamaFileType.MOSTLY_Q2_K_S,
+ hparams,
+ n_dims=2,
+ )
+
+ assert qtype == gguf.GGMLQuantizationType.Q4_K
+
+
+def test_mtp_ffn_down_dtype_follows_llama_cpp_mixed_rule():
+ hparams = {"num_hidden_layers": 28, "mtp_num_hidden_layers": 1, "num_attention_heads": 16, "num_key_value_heads": 4}
+
+ qtype = select_llama_cpp_compatible_qtype(
+ "blk.28.ffn_down.weight",
+ gguf.LlamaFileType.MOSTLY_Q4_K_M,
+ hparams,
+ n_dims=2,
+ )
+
+ assert qtype == gguf.GGMLQuantizationType.Q6_K
+
+
+def test_mtp_attn_output_dtype_follows_llama_cpp_mixed_rule():
+ hparams = {"num_hidden_layers": 28, "mtp_num_hidden_layers": 1, "num_attention_heads": 16, "num_key_value_heads": 4}
+
+ qtype = select_llama_cpp_compatible_qtype(
+ "blk.28.attn_output.weight",
+ gguf.LlamaFileType.MOSTLY_Q3_K_M,
+ hparams,
+ n_dims=2,
+ )
+
+ assert qtype == gguf.GGMLQuantizationType.Q4_K
+
+
+def test_one_dim_mtp_tensors_stay_f32():
+ hparams = {"num_hidden_layers": 28}
+
+ qtype = select_llama_cpp_compatible_qtype(
+ "blk.28.ffn_norm.weight",
+ gguf.LlamaFileType.MOSTLY_Q4_K_M,
+ hparams,
+ n_dims=1,
+ )
+
+ assert qtype == gguf.GGMLQuantizationType.F32
+
+
+def test_selector_tracks_attn_v_counter_for_q3_k_m():
+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}
+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q3_k_m"))
+
+ qtypes = [selector.select_qtype(f"blk.{idx}.attn_v.weight", n_dims=2) for idx in range(3)]
+
+ assert qtypes == [
+ gguf.GGMLQuantizationType.Q5_K,
+ gguf.GGMLQuantizationType.Q5_K,
+ gguf.GGMLQuantizationType.Q4_K,
+ ]
+
+
+def test_selector_returns_gguf_type_string_for_compressor_config():
+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}
+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q2_k_s"))
+
+ assert selector.select_gguf_type("blk.28.attn_v.weight", n_dims=2) == "gguf:q4_k"
+
+
+def test_attn_qkv_matches_llama_cpp_attn_v_like_rule():
+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}
+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q3_k_m"))
+
+ qtype = selector.select_qtype("blk.0.attn_qkv.weight", n_dims=2)
+
+ assert qtype == gguf.GGMLQuantizationType.Q5_K
+ assert selector.i_attention_wv == 1
+
+
+def test_selector_accepts_explicit_layer_count_from_compressor():
+ hparams = {"num_hidden_layers": 2, "num_attention_heads": 16, "num_key_value_heads": 4}
+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_k_m"), n_layer=28)
+
+ assert selector.select_gguf_type("blk.1.attn_v.weight", n_dims=2) == "gguf:q6_k"
+
+
+def test_selector_uses_cpp_integer_layer_division():
+ hparams = {"num_hidden_layers": 2, "num_attention_heads": 16, "num_key_value_heads": 4}
+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_k_m"), n_layer=2)
+
+ assert selector.select_gguf_type("blk.1.ffn_down.weight", n_dims=2) == "gguf:q6_k"
+
+
+def test_attn_v_more_bits_uses_attention_counter_not_layer_id():
+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}
+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_k_m"), n_attention_wv=2)
+
+ assert selector.select_gguf_type("blk.0.attn_v.weight", n_dims=2) == "gguf:q4_k"
+ assert selector.select_gguf_type("blk.1.attn_v.weight", n_dims=2) == "gguf:q6_k"
+
+
+def test_q4_0_ffn_down_upgrade_requires_imatrix_like_llama_cpp():
+ hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}
+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_0"), has_imatrix=False)
+
+ assert selector.select_gguf_type("blk.0.ffn_down.weight", n_dims=2) == "gguf:q4_0"
diff --git a/test/test_cpu/schemes/test_scheme.py b/test/test_cpu/schemes/test_scheme.py
index b011d7fc0..3c64be8b9 100644
--- a/test/test_cpu/schemes/test_scheme.py
+++ b/test/test_cpu/schemes/test_scheme.py
@@ -2,10 +2,12 @@ import os
import shutil
import pytest
+import torch
+import torch.nn as nn
import transformers
from auto_round import AutoRound
-from auto_round.schemes import QuantizationScheme
+from auto_round.schemes import QuantizationScheme, _handle_special_schemes
from ...helpers import get_model_path, get_tiny_model, opt_name_or_path, qwen_name_or_path, save_tiny_model
@@ -229,3 +231,86 @@ class TestAutoRound:
_, quantized_model_path = ar.quantize_and_save(self.save_folder)
model = transformers.AutoModelForCausalLM.from_pretrained(quantized_model_path, trust_remote_code=True)
assert model is not None, "Model loading failed after quantization with FP8_STATIC scheme"
+
+
+class TestQ2KMixedRecipe:
+ """Unit tests for the gguf:q2_k_mixed _handle_special_schemes recipe.
+
+ Uses a small dummy model so no real checkpoint is required.
+ """
+
+ class _DummyMLP(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.up_proj = nn.Linear(8, 8)
+ self.down_proj = nn.Linear(8, 8)
+ self.gate_proj = nn.Linear(8, 8)
+
+ class _DummyMoE(nn.Module):
+ def __init__(self):
+ super().__init__()
+ # singular form used by Qwen2MoE / Qwen3.6 / Qwen3_5MoE
+ self.shared_expert = TestQ2KMixedRecipe._DummyMLP()
+ self.shared_expert_gate = nn.Linear(8, 1)
+ # plural form used by DeepSeek / Mistral-Large
+ self.shared_experts = TestQ2KMixedRecipe._DummyMLP()
+ # regular routed experts (should NOT be promoted)
+ self.experts = nn.ModuleList([nn.Linear(8, 8) for _ in range(2)])
+ # fused expert weights stay 3D and should remain q2k in GGUF mapping
+ self.gate_up_proj = nn.Linear(8, 16, bias=False)
+ self.gate_up_proj.weight = nn.Parameter(torch.zeros(2, 16, 8))
+ self.down_proj = nn.Linear(8, 8, bias=False)
+ self.down_proj.weight = nn.Parameter(torch.zeros(2, 8, 8))
+
+ class _DummyModel(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layer0 = nn.Module()
+ self.layer0.mlp = TestQ2KMixedRecipe._DummyMoE()
+ # lm_head is Linear but should get Q8_0, not Q4_K_S
+ self.lm_head = nn.Linear(8, 8)
+ # embedding should get Q8_0
+ self.embed_tokens = nn.Embedding(16, 8)
+
+ def setup_method(self):
+ self.model = self._DummyModel()
+ self.cfg = _handle_special_schemes("gguf:q2_k_mixed", {}, self.model)
+
+ def test_singular_shared_expert_linear_gets_q4ks(self):
+ """2D shared expert linears should use Q4_K_S."""
+ for sub in ("up_proj", "down_proj", "gate_proj"):
+ key = f"layer0.mlp.shared_expert.{sub}"
+ assert key in self.cfg, f"{key} missing from layer_config"
+ assert self.cfg[key] == "GGUF:Q4_K_S", f"{key} expected Q4_K_S, got {self.cfg[key]}"
+
+ def test_shared_expert_gate_gets_q4ks(self):
+ """2D shared expert gate should use Q4_K_S."""
+ key = "layer0.mlp.shared_expert_gate"
+ assert key in self.cfg, f"{key} missing from layer_config"
+ assert self.cfg[key] == "GGUF:Q4_K_S", f"{key} expected Q4_K_S, got {self.cfg[key]}"
+
+ def test_plural_shared_experts_linear_gets_q4ks(self):
+ """2D plural shared experts should also use Q4_K_S."""
+ for sub in ("up_proj", "down_proj", "gate_proj"):
+ key = f"layer0.mlp.shared_experts.{sub}"
+ assert key in self.cfg, f"{key} missing from layer_config"
+ assert self.cfg[key] == "GGUF:Q4_K_S", f"{key} expected Q4_K_S, got {self.cfg[key]}"
+
+ def test_routed_experts_linears_not_in_recipe(self):
+ """Routed expert linears are excluded from the recipe — falls through to dtype selector."""
+ for i in range(2):
+ key = f"layer0.mlp.experts.{i}"
+ assert key not in self.cfg, f"{key} should not be in the recipe"
+
+ def test_fused_three_dim_expert_weights_start_as_q2ks_recipe(self):
+ """_handle_special_schemes sets 3D expert weights directly to Q2_K_S."""
+ assert self.cfg.get("layer0.mlp.gate_up_proj") == "GGUF:Q2_K_S"
+ assert self.cfg.get("layer0.mlp.down_proj") == "GGUF:Q2_K_S"
+
+ def test_lm_head_gets_q8_0(self):
+ """lm_head should be Q8_0."""
+ assert self.cfg.get("lm_head") == "GGUF:Q8_0"
+
+ def test_embedding_gets_q8_0(self):
+ """embed_tokens embedding should be Q8_0."""
+ assert self.cfg.get("embed_tokens") == "GGUF:Q8_0"
diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.py
index fb4069048..458266b86 100644
--- a/test/test_cuda/export/test_gguf_format.py
+++ b/test/test_cuda/export/test_gguf_format.py
@@ -177,7 +177,7 @@ class TestAutoRound:
if "mmproj-model.gguf" in file:
assert abs(file_size - 75) < 5.0
else:
- assert abs(file_size - 683) < 5.0
+ assert abs(file_size - 690) < 10.0
shutil.rmtree(tiny_model_path, ignore_errors=True)