PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix the mismatch between weight and scale in gguf format (#2014)
Heng Guob482d1507702
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex 3ac20abea..fdd53b7bc 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -750,6 +750,34 @@ def _gguf_type_fallback(gguf_type: str) -> str:return gguf_type+def _resolve_gguf_n_layers(config, model_type=ModelType.TEXT):+ """Resolve text (and vision) layer counts from a model config.++ Multimodal configs (e.g. Qwen3.5, Qwen3-VL) keep the text hparams only in+ ``config.text_config``, so both it and the top-level config must be checked.+ """+ n_layer = None+ n_layer_vision = None+ text_config = getattr(config, "text_config", None)+ for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:+ if hasattr(config, name):+ n_layer = getattr(config, name)+ if text_config is not None and hasattr(text_config, name):+ # the text hparams in text_config are authoritative for multimodal models;+ # for text-only usage they still fill in when the top-level config lacks them+ if model_type != ModelType.TEXT or n_layer is None:+ n_layer = getattr(text_config, name)+ if model_type != ModelType.TEXT:+ for config_name in ["vision_config", "vision_encoder"]:+ if hasattr(config, config_name):+ if hasattr(getattr(config, config_name), name):+ n_layer_vision = getattr(getattr(config, config_name), name)+ break+ if n_layer and n_layer_vision:+ break+ return n_layer, n_layer_vision++##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT):# # TODO: support for other format later@@ -782,23 +810,7 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, modelexcept NotImplementedError:return layer_config, {}- n_layer = None- if model_type != ModelType.TEXT:- n_layer_vision = None- for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:- if hasattr(model.config, name):- n_layer = getattr(model.config, name)- if model_type != ModelType.TEXT:- if n_layer is not None and hasattr(model.config, "text_config"):- if hasattr(getattr(model.config, "text_config"), name):- n_layer = getattr(getattr(model.config, "text_config"), name)- for config_name in ["vision_config", "vision_encoder"]:- if hasattr(model.config, config_name):- if hasattr(getattr(model.config, config_name), name):- n_layer_vision = getattr(getattr(model.config, config_name), name)- break- if n_layer and n_layer_vision:- break+ n_layer, n_layer_vision = _resolve_gguf_n_layers(model.config, model_type)if n_layer is None:return layer_config, {}diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 3c40d18be..a2a1bfa97 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -68,12 +68,19 @@ if TYPE_CHECKING:def wrapper_model_instance(- model_instance, model, layer_config, low_cpu_mem_usage=False, device=None, quant_nontext_module=False+ model_instance,+ model,+ layer_config,+ low_cpu_mem_usage=False,+ device=None,+ quant_nontext_module=False,+ is_auto_scheme=False,):if model_instance.model_arch == gguf.MODEL_ARCH.MMPROJ and model_instance.fname_out.is_dir():model_instance.fname_out = model_instance.fname_out / "mmproj-model.gguf"model_instance.model = modelmodel_instance.layer_config = layer_config+ model_instance.is_auto_scheme = is_auto_schememodel_instance.low_cpu_mem_usage = _need_low_cpu_mem(low_cpu_mem_usage)model_instance._gguf_written_hf_names = set()model_instance._gguf_written_checkpoint_names = set()@@ -221,15 +228,160 @@ def _quant_data_with_args(return data-def _remap_quant_attr_tensor(cls, attr_tensor, module, modify_name, new_name, bid):- if not isinstance(attr_tensor, torch.Tensor):- return attr_tensor- if not (hasattr(cls, "permute") or need_modify_tensor(cls, modify_name)):- return attr_tensor+def _tensors_equal(a, b):+ if a.shape != b.shape:+ return False+ if a.dtype != b.dtype:+ b = b.to(a.dtype)+ if a.device != b.device:+ b = b.to(a.device)+ return torch.equal(a, b)+++def _trace_modify_transform(cls, in_shape, modify_name, new_name, bid, out_shape):+ """Probe ``cls.modify_tensors`` with index tensors to identify the weight transform.++ Runs the transform on row-index and column-index tensors (each twice, with an+ affine pair ``2*x+1`` to reject arithmetic transforms that would fake a valid+ index map). Returns ``(rowmap, colmap)`` mapping every output row/column back+ to its source row/column when the transform is a pure gather that keeps rows+ and columns independent, else None.+ """+ rows, cols = in_shape++ def run(probe):+ outputs = dict(cls.modify_tensors(probe, modify_name, bid))+ out = outputs.get(new_name)+ if out is None or tuple(out.shape) != tuple(out_shape):+ return None+ return out.to(torch.float32)++ try:+ row_probe = torch.arange(rows, dtype=torch.float32).unsqueeze(1).expand(rows, cols).contiguous()+ out_r = run(row_probe)+ out_r2 = run(row_probe * 2 + 1)+ del row_probe+ if out_r is None or out_r2 is None or not torch.equal(out_r2, out_r * 2 + 1):+ return None # not a pure gather (arithmetic transform)+ if not torch.equal(out_r, out_r[:, :1].expand_as(out_r)):+ return None # mixes different source rows into one output row+ del out_r2++ col_probe = torch.arange(cols, dtype=torch.float32).unsqueeze(0).expand(rows, cols).contiguous()+ out_c = run(col_probe)+ out_c2 = run(col_probe * 2 + 1)+ del col_probe+ if out_c is None or out_c2 is None or not torch.equal(out_c2, out_c * 2 + 1):+ return None+ if not torch.equal(out_c, out_c[:1, :].expand_as(out_c)):+ return None # mixes different source columns into one output column++ rowmap = out_r[:, 0].round().to(torch.long)+ colmap = out_c[0].round().to(torch.long)+ except Exception as e:+ logger.debug("transform tracing failed for %s: %s", new_name, e)+ return None+ if rowmap.numel() == 0 or rowmap.min() < 0 or rowmap.max() >= rows:+ return None+ if colmap.numel() == 0 or colmap.min() < 0 or colmap.max() >= cols:+ return None+ return rowmap, colmap+++def _remap_traced_attrs(kwargs, rowmap, colmap, in_shape):+ """Apply a traced (rowmap, colmap) gather to per-group quant params.- bs = module.weight.shape[0]- remapped = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)).get(new_name)- return attr_tensor if remapped is None else remapped+ Quant params are row-major with one or more values per group of weight+ columns, so row gathers always transfer; column gathers transfer only when+ they move whole param groups. Returns the remapped params, or None when any+ present param cannot be remapped (params must be used all-or-nothing).+ """+ in_rows, in_cols = in_shape+ col_identity = colmap.numel() == in_cols and torch.equal(colmap, torch.arange(in_cols, dtype=colmap.dtype))+ remapped = {}+ for key, t in kwargs.items():+ if t is None or not isinstance(t, torch.Tensor):+ remapped[key] = t+ continue+ if key == "imatrix":+ # imatrix is per input column and unaffected by row gathers+ if col_identity:+ remapped[key] = t+ elif t.dim() == 1 and t.numel() == in_cols:+ remapped[key] = t[colmap.to(t.device)]+ else:+ return None+ continue+ if t.numel() == 0 or t.numel() % in_rows != 0:+ return None+ m = t.reshape(in_rows, -1)+ m = m.index_select(0, rowmap.to(t.device))+ if not col_identity:+ per_row = m.shape[1]+ if per_row == 0 or in_cols % per_row != 0:+ return None+ group_size = in_cols // per_row # weight columns covered by one param value+ if colmap.numel() % group_size != 0:+ return None+ groups = colmap.reshape(-1, group_size)+ starts = groups[:, 0]+ expected = starts.unsqueeze(1) + torch.arange(group_size, dtype=colmap.dtype)+ if not (torch.equal(groups, expected) and bool((starts % group_size == 0).all())):+ return None # column moves cross param-group boundaries+ m = m.index_select(1, (starts // group_size).to(t.device))+ remapped[key] = m+ return remapped+++def _adapt_quant_attrs_to_transform(cls, module, data_torch, modify_name, new_name, bid, kwargs):+ """Make the module's quant params (stored in the original HF weight layout)+ valid for the tensor produced by ``modify_tensors``.++ Returns the (possibly remapped) params, or None when the transform cannot be+ applied to them — the caller must then drop the params so ``ggml_quant``+ re-searches them on the transformed weights.+ """+ weight = getattr(module, "weight", None)+ if weight is None or not isinstance(weight, torch.Tensor):+ return None+ if _tensors_equal(data_torch, weight):+ return kwargs+ if weight.dim() != 2 or data_torch.dim() != 2:+ return None+ trace = _trace_modify_transform(cls, tuple(weight.shape), modify_name, new_name, bid, tuple(data_torch.shape))+ if trace is None:+ return None+ return _remap_traced_attrs(kwargs, trace[0], trace[1], tuple(weight.shape))+++def _verify_packed_tensor(data, data_qtype, ref, new_name):+ """Guard against packing with mismatched quant params: dequantize a sample of+ the packed tensor and compare against the (already fake-quantized) weights.+ When the provided params match the tensor layout the round-trip is near-exact,+ so any sizable deviation means the export would be silently corrupted.++ ``ref`` must be a snapshot of the leading weight rows taken BEFORE ggml_quant,+ whose kernels modify their input buffer in place."""+ try:+ arr = data if isinstance(data, np.ndarray) else np.asarray(data)+ if arr.ndim < 2 or ref.dim() < 2:+ return+ k = min(arr.shape[0], ref.shape[0])+ dequantized = gguf.quants.dequantize(arr[:k], data_qtype)+ ref = ref[:k].detach().to(torch.float32).cpu().numpy()+ if dequantized.shape != ref.shape:+ return+ rel_err = float(np.linalg.norm(dequantized - ref)) / max(float(np.linalg.norm(ref)), 1e-12)+ except Exception as e:+ logger.debug("pack verification skipped for %s: %s", new_name, e)+ return+ if rel_err > 0.1:+ raise ValueError(+ f"{new_name}: packed tensor deviates from the quantized weights by {rel_err:.1%}; "+ "the quantization params do not match the tensor layout, the export would be corrupted"+ )+ if rel_err > 0.02:+ logger.warning(f"{new_name}: packed tensor deviates from the quantized weights by {rel_err:.1%}")def _expected_scale_columns(ggml_type):@@ -273,13 +425,6 @@ def _validate_quant_attr_shapes(data_torch, data_qtype, kwargs, name, new_name):)-def need_modify_tensor(cls, name):- hf_arch = getattr(cls, "hf_arch", "")- if hf_arch in "Qwen3NextForCausalLM" and "in_proj_qkvz.weight" in name:- return True- return False--def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=None, use_layer_attrs=True):"""@@ -313,48 +458,29 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d"wmin": module.w_wmin if hasattr(module, "w_wmin") else None,"imatrix": module.imatrix if hasattr(module, "imatrix") else None,}- # patch for Qwen3_5, Qwen3_5 handles some weights specially,- # but the scale doesn't match; these weights are handled by gguf itself.- # Define model architectures that need special handling- QWEN3_5_MODELS = {- "Qwen3_5ForCausalLM",- "Qwen3_5MoeForCausalLM",- "Qwen3_5MoeForConditionalGeneration",- "Qwen3_5ForConditionalGeneration",- }-- QWEN3_5_SKIP_KEYS = {- ".in_proj_qkv.",- ".in_proj_z",- ".conv1d",- ".in_proj_b.",- ".in_proj_a.",- ".A_log",- ".dt_bias",- ".out_proj.",- }-- hf_arch = getattr(cls, "hf_arch", "")- should_skip = hf_arch in QWEN3_5_MODELS and any(key in name for key in QWEN3_5_SKIP_KEYS)-- if use_layer_attrs and not should_skip:- # support for MOE model with cls experts not linear- for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:- if not (hasattr(module, attr) and getattr(module, attr) is not None):- continue-- attr_tensor = getattr(module, attr)- if not isinstance(attr_tensor, torch.Tensor):- continue-- attr_tensor = _remap_quant_attr_tensor(cls, attr_tensor, module, modify_name, new_name, bid)-- # Map attribute names to kwargs keys: w_d_scale -> d_scale, w_d_wmin -> d_wmin, w_wmin -> wmin- kwargs_key = attr.replace("w_", "") if attr.startswith("w_") else attr- kwargs[kwargs_key] = attr_tensor.to(torch.float32)+ if any(isinstance(v, torch.Tensor) for v in kwargs.values()):+ # modify_tensors may have permuted/split the weights (e.g. llama Q/K+ # interleave, Qwen3.5 V-head reorder); the stored quant params must+ # follow the same transform or they describe the wrong rows/columns.+ adapted = _adapt_quant_attrs_to_transform(cls, module, data_torch, modify_name, new_name, bid, kwargs)+ if adapted is None:+ logger.info(+ f"{new_name}: weight transform is incompatible with the stored quantization params,"+ " re-searching them on the transformed weights"+ )+ adapted = dict.fromkeys(kwargs)+ kwargs = {+ k: v.to(torch.float32) if isinstance(v, torch.Tensor) and k != "imatrix" else v+ for k, v in adapted.items()+ }data_torch = data_torch.to(torch.float32)_validate_quant_attr_shapes(data_torch, data_qtype, kwargs, name, new_name)+ # ggml_quant kernels modify their input buffer in place; snapshot the rows+ # used by the post-pack verification first.+ verify_ref = data_torch[:64].detach().clone() if kwargs.get("scale") is not None and data_torch.dim() >= 2 else Nonedata = ggml_quant(data_torch, data_qtype.name.lower(), device=device, **kwargs)+ if verify_ref is not None:+ _verify_packed_tensor(data, data_qtype, verify_ref, new_name)# else:# # if data_torch.dtype ==torch.float32:# # data_qtype = gguf.GGMLQuantizationType.F32@@ -437,6 +563,34 @@ def _qtype_name(qtype):return qtype.name if hasattr(qtype, "name") else str(qtype)+def _qtype_precision_rank(qtype):+ ranks = {+ gguf.GGMLQuantizationType.Q2_K: 2,+ gguf.GGMLQuantizationType.Q3_K: 3,+ gguf.GGMLQuantizationType.Q4_0: 4,+ gguf.GGMLQuantizationType.Q4_1: 4,+ gguf.GGMLQuantizationType.Q4_K: 4,+ gguf.GGMLQuantizationType.Q5_0: 5,+ gguf.GGMLQuantizationType.Q5_1: 5,+ gguf.GGMLQuantizationType.Q5_K: 5,+ gguf.GGMLQuantizationType.Q6_K: 6,+ gguf.GGMLQuantizationType.Q8_0: 8,+ gguf.GGMLQuantizationType.Q8_1: 8,+ gguf.GGMLQuantizationType.Q8_K: 8,+ gguf.GGMLQuantizationType.F16: 16,+ gguf.GGMLQuantizationType.BF16: 16,+ gguf.GGMLQuantizationType.F32: 32,+ }+ return ranks.get(qtype, 0)+++def _should_keep_recipe_qtype(layer_config_qtype, fallback_qtype, allow_recipe_fallback):+ """Keep llama.cpp fixed-format recipe upgrades unless AutoScheme owns dtype selection."""+ if not allow_recipe_fallback or layer_config_qtype is None:+ return False+ return _qtype_precision_rank(fallback_qtype) > _qtype_precision_rank(layer_config_qtype)++def resolve_restored_qtype(layer_config,hf_names,@@ -444,6 +598,7 @@ def resolve_restored_qtype(gguf_name,fallback_qtype,diagnostics,+ allow_recipe_fallback=False,):source_qtypes = [get_qtype_by_layer_config(layer_config, hf_name, fallback_qtype, explicit_only=True) for hf_name in hf_names@@ -451,10 +606,14 @@ def resolve_restored_qtype(matched_qtypes = [qtype for qtype in source_qtypes if qtype is not None]if len(matched_qtypes) == 1 and len(hf_names) == 1:+ if _should_keep_recipe_qtype(matched_qtypes[0], fallback_qtype, allow_recipe_fallback):+ return Nonereturn matched_qtypes[0]if len(matched_qtypes) > 0 and len(matched_qtypes) == len(source_qtypes):unique_qtypes = set(matched_qtypes)if len(unique_qtypes) == 1:+ if _should_keep_recipe_qtype(matched_qtypes[0], fallback_qtype, allow_recipe_fallback):+ return Nonereturn matched_qtypes[0]if len(hf_names) <= 1 and len(matched_qtypes) == 0:@@ -747,6 +906,7 @@ def prepare_tensors(cls):if restored.transform_kind != "extra" and new_name.endswith(".weight") and n_dims > 1else []),+ allow_recipe_fallback=not getattr(cls, "is_auto_scheme", False),)# # No override (data_qtype is False), or wants to be quantized (data_qtype is True)if layer_config_qtype is not None:diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex bf9d36c0a..e5c410cca 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -83,6 +83,7 @@ def create_model_class(model_type=ModelType.TEXT,device="cpu",quant_nontext_module: bool = False,+ is_auto_scheme: bool = False,):tmp_work_dir = model.name_or_pathos.makedirs(output_dir, exist_ok=True)@@ -134,6 +135,7 @@ def create_model_class(low_cpu_mem_usage=low_cpu_mem_usage,device=device,quant_nontext_module=quant_nontext_module,+ is_auto_scheme=is_auto_scheme,)model_instance = handle_special_model(model_instance, model_architecture)return model_instance@@ -152,6 +154,7 @@ def pack_gguf_layer(model_type=ModelType.TEXT,device="cpu",quant_nontext_module=False,+ is_auto_scheme=False,):"""Export the model to gguf format."""global gguf_model_instance_global@@ -166,6 +169,7 @@ def pack_gguf_layer(model_type=ModelType.TEXT,device=device,quant_nontext_module=quant_nontext_module,+ is_auto_scheme=is_auto_scheme,)]if model_type == ModelType.MMPROJ:@@ -179,6 +183,7 @@ def pack_gguf_layer(model_type=ModelType.MMPROJ,device=device,quant_nontext_module=quant_nontext_module,+ is_auto_scheme=is_auto_scheme,))@@ -236,6 +241,7 @@ def save_quantized_as_gguf(mllm=False,device="cpu",quant_nontext_module=False,+ is_auto_scheme=False,**kwargs,):"""Export the model to gguf format."""@@ -252,6 +258,7 @@ def save_quantized_as_gguf(model_type=ModelType.TEXT,device=device,quant_nontext_module=quant_nontext_module,+ is_auto_scheme=is_auto_scheme,)]if mllm:@@ -264,6 +271,7 @@ def save_quantized_as_gguf(model_type=ModelType.MMPROJ,device=device,quant_nontext_module=quant_nontext_module,+ is_auto_scheme=is_auto_scheme,))diff --git a/auto_round/export/export_to_gguf/gguf_dtype.py b/auto_round/export/export_to_gguf/gguf_dtype.pyindex 1817ad46d..6c173d3de 100644--- a/auto_round/export/export_to_gguf/gguf_dtype.py+++ b/auto_round/export/export_to_gguf/gguf_dtype.py@@ -223,7 +223,6 @@ class GGUFDTypeSelector:)or 0)-# llama.cpp: output & token_embd handling (llama_tensor_get_type outer wrapper)if category == TensorCategory.OUTPUT or (self.has_tied_embeddings and category == TensorCategory.TOKEN_EMBD):# llama.cpp: if new_type != Q8_0, upgrade to Q6_K@@ -266,7 +265,7 @@ class GGUFDTypeSelector:elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M:if i_layer < n_layer // 16:qtype = gguf.GGMLQuantizationType.Q5_K- elif self.model_arch == gguf.MODEL_ARCH.FALCON or _use_more_bits(i_layer, n_layer):+ elif self.model_arch != gguf.MODEL_ARCH.FALCON or _use_more_bits(i_layer, n_layer):qtype = gguf.GGMLQuantizationType.Q4_Kelse:qtype = gguf.GGMLQuantizationType.Q3_Kdiff --git a/auto_round/formats.py b/auto_round/formats.pyindex d16de4756..9dd9ce21d 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -783,6 +783,7 @@ class GGUFFormat(OutputFormat):format_name = "gguf"def __init__(self, format: str, ar: BaseCompressor):+ self.is_auto_scheme = bool(getattr(ar, "is_auto_scheme", False))if format.startswith("gguf:"):self._original_format = format # preserve "gguf:q2_k_mixed" etc. for Phase 2bself.output_format = "gguf"@@ -875,6 +876,7 @@ class GGUFFormat(OutputFormat):model_type,device,quant_nontext_module,+ is_auto_scheme=self.is_auto_scheme,)def save_quantized(@@ -899,6 +901,7 @@ class GGUFFormat(OutputFormat):mllm=self.mllm,device=device,serialization_dict=serialization_dict,+ is_auto_scheme=self.is_auto_scheme,**kwargs,)diff --git a/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.pyindex d06c1f278..3f76bee91 100644--- a/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py+++ b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py@@ -115,7 +115,7 @@ def test_restorer_does_not_rerun_completed_conversion_group(monkeypatch):assert restored == []-def test_resolve_restored_qtype_uses_consistent_multi_source_layer_config():+def test_resolve_restored_qtype_keeps_higher_recipe_fallback_for_multi_source_layer_config():import gguffrom auto_round.export.export_to_gguf.convert import resolve_restored_qtype@@ -134,12 +134,60 @@ def test_resolve_restored_qtype_uses_consistent_multi_source_layer_config():gguf_name="blk.0.ffn_gate_up.weight",fallback_qtype=gguf.GGMLQuantizationType.Q5_K,diagnostics=diagnostics,+ allow_recipe_fallback=True,+ )++ assert qtype is None+ assert diagnostics == []+++def test_resolve_restored_qtype_respects_mixed_layer_config_even_when_recipe_is_higher():+ import gguf++ from auto_round.export.export_to_gguf.convert import resolve_restored_qtype++ diagnostics = []+ qtype = resolve_restored_qtype(+ layer_config={+ "model.layers.0.mlp.gate_proj": {"bits": 4, "super_bits": 6, "sym": False},+ "model.layers.0.mlp.up_proj": {"bits": 4, "super_bits": 6, "sym": False},+ },+ hf_names=(+ "model.layers.0.mlp.gate_proj.weight",+ "model.layers.0.mlp.up_proj.weight",+ ),+ checkpoint_name="model.layers.0.mlp.gate_up_proj.weight",+ gguf_name="blk.0.ffn_gate_up.weight",+ fallback_qtype=gguf.GGMLQuantizationType.Q5_K,+ diagnostics=diagnostics,+ allow_recipe_fallback=False,)assert qtype == gguf.GGMLQuantizationType.Q4_Kassert diagnostics == []+def test_resolve_restored_qtype_keeps_higher_layer_config_than_recipe_fallback():+ import gguf++ from auto_round.export.export_to_gguf.convert import resolve_restored_qtype++ diagnostics = []+ qtype = resolve_restored_qtype(+ layer_config={+ "model.layers.0.mlp.down_proj": {"bits": 6, "super_bits": 8, "sym": True},+ },+ hf_names=("model.layers.0.mlp.down_proj.weight",),+ checkpoint_name="model.layers.0.mlp.down_proj.weight",+ gguf_name="blk.0.ffn_down.weight",+ fallback_qtype=gguf.GGMLQuantizationType.Q5_K,+ diagnostics=diagnostics,+ )++ assert qtype == gguf.GGMLQuantizationType.Q6_K+ assert diagnostics == []++def test_resolve_restored_qtype_falls_back_and_records_conflicting_multi_source_layer_config():import ggufdiff --git a/test/test_cpu/export/test_gguf_mtp_dtype.py b/test/test_cpu/export/test_gguf_mtp_dtype.pyindex a036b8a0e..6cad988c2 100644--- a/test/test_cpu/export/test_gguf_mtp_dtype.py+++ b/test/test_cpu/export/test_gguf_mtp_dtype.py@@ -1,11 +1,15 @@+from types import SimpleNamespace+import pytest+from auto_round.compressors.utils import _resolve_gguf_n_layersfrom auto_round.export.export_to_gguf.gguf_dtype import (GGUFDTypeSelector,gguf_format_to_ftype,select_llama_cpp_compatible_qtype,)from auto_round.utils import LazyImport+from auto_round.utils.model import ModelTypegguf = LazyImport("gguf")@@ -115,8 +119,44 @@ def test_attn_v_more_bits_uses_attention_counter_not_layer_id():assert selector.select_gguf_type("blk.1.attn_v.weight", n_dims=2) == "gguf:q6_k"+def test_q3_k_m_ffn_down_uses_q4_k_for_non_falcon_like_llama_cpp():+ # llama.cpp: new_type = i_layer < n_layer/16 ? Q5_K+ # : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? Q4_K : Q3_K+ # so for non-Falcon archs every ffn_down beyond the first n_layer/16 layers is Q4_K.+ hparams = {"num_hidden_layers": 33, "num_attention_heads": 16, "num_key_value_heads": 4}+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q3_k_m"))++ qtypes = {i: selector.select_qtype(f"blk.{i}.ffn_down.weight", n_dims=2) for i in range(33)}++ assert qtypes[0] == qtypes[1] == gguf.GGMLQuantizationType.Q5_K+ assert all(qtypes[i] == gguf.GGMLQuantizationType.Q4_K for i in range(2, 33))++def test_q4_0_ffn_down_upgrade_requires_imatrix_like_llama_cpp():hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_0"), has_imatrix=False)assert selector.select_gguf_type("blk.0.ffn_down.weight", n_dims=2) == "gguf:q4_0"+++def test_n_layer_resolution_reads_nested_text_config():+ # multimodal configs like Qwen3.5 expose num_hidden_layers only under text_config;+ # failing to resolve it silently disabled the whole llama.cpp k-quant mixture.+ config = SimpleNamespace(+ text_config=SimpleNamespace(num_hidden_layers=32),+ vision_config=SimpleNamespace(depth=24),+ )++ assert _resolve_gguf_n_layers(config, ModelType.TEXT) == (32, None)+ assert _resolve_gguf_n_layers(config, ModelType.MMPROJ) == (32, 24)+++def test_n_layer_resolution_prefers_text_config_for_multimodal():+ config = SimpleNamespace(+ num_hidden_layers=48,+ text_config=SimpleNamespace(num_hidden_layers=32),+ vision_config=SimpleNamespace(depth=24),+ )++ assert _resolve_gguf_n_layers(config, ModelType.MMPROJ) == (32, 24)+ assert _resolve_gguf_n_layers(config, ModelType.TEXT) == (48, None)diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.pyindex 2c76f0671..458266b86 100644--- a/test/test_cuda/export/test_gguf_format.py+++ b/test/test_cuda/export/test_gguf_format.py@@ -175,7 +175,7 @@ class TestAutoRound:print(f"{file}: {os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2} MB")file_size = os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2if "mmproj-model.gguf" in file:- assert abs(file_size - 62) < 5.0+ assert abs(file_size - 75) < 5.0else:assert abs(file_size - 690) < 10.0