Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix the mismatch between weight and scale in gguf format (#2014)

Heng Guob482d1507702
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index 3ac20abea..fdd53b7bc 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -750,6 +750,34 @@ def _gguf_type_fallback(gguf_type: str) -> str:
return gguf_type
+def _resolve_gguf_n_layers(config, model_type=ModelType.TEXT):
+ """Resolve text (and vision) layer counts from a model config.
+
+ Multimodal configs (e.g. Qwen3.5, Qwen3-VL) keep the text hparams only in
+ ``config.text_config``, so both it and the top-level config must be checked.
+ """
+ n_layer = None
+ n_layer_vision = None
+ text_config = getattr(config, "text_config", None)
+ for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:
+ if hasattr(config, name):
+ n_layer = getattr(config, name)
+ if text_config is not None and hasattr(text_config, name):
+ # the text hparams in text_config are authoritative for multimodal models;
+ # for text-only usage they still fill in when the top-level config lacks them
+ if model_type != ModelType.TEXT or n_layer is None:
+ n_layer = getattr(text_config, name)
+ if model_type != ModelType.TEXT:
+ for config_name in ["vision_config", "vision_encoder"]:
+ if hasattr(config, config_name):
+ if hasattr(getattr(config, config_name), name):
+ n_layer_vision = getattr(getattr(config, config_name), name)
+ break
+ if n_layer and n_layer_vision:
+ break
+ return n_layer, n_layer_vision
+
+
##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129
def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model, model_type=ModelType.TEXT):
# # TODO: support for other format later
@@ -782,23 +810,7 @@ def get_layer_config_by_gguf_format(layer_config, target_gguf_format: str, model
except NotImplementedError:
return layer_config, {}
- n_layer = None
- if model_type != ModelType.TEXT:
- n_layer_vision = None
- for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers", "depth"]:
- if hasattr(model.config, name):
- n_layer = getattr(model.config, name)
- if model_type != ModelType.TEXT:
- if n_layer is not None and hasattr(model.config, "text_config"):
- if hasattr(getattr(model.config, "text_config"), name):
- n_layer = getattr(getattr(model.config, "text_config"), name)
- for config_name in ["vision_config", "vision_encoder"]:
- if hasattr(model.config, config_name):
- if hasattr(getattr(model.config, config_name), name):
- n_layer_vision = getattr(getattr(model.config, config_name), name)
- break
- if n_layer and n_layer_vision:
- break
+ n_layer, n_layer_vision = _resolve_gguf_n_layers(model.config, model_type)
if n_layer is None:
return layer_config, {}
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 3c40d18be..a2a1bfa97 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -68,12 +68,19 @@ if TYPE_CHECKING:
def wrapper_model_instance(
- model_instance, model, layer_config, low_cpu_mem_usage=False, device=None, quant_nontext_module=False
+ model_instance,
+ model,
+ layer_config,
+ low_cpu_mem_usage=False,
+ device=None,
+ quant_nontext_module=False,
+ is_auto_scheme=False,
):
if model_instance.model_arch == gguf.MODEL_ARCH.MMPROJ and model_instance.fname_out.is_dir():
model_instance.fname_out = model_instance.fname_out / "mmproj-model.gguf"
model_instance.model = model
model_instance.layer_config = layer_config
+ model_instance.is_auto_scheme = is_auto_scheme
model_instance.low_cpu_mem_usage = _need_low_cpu_mem(low_cpu_mem_usage)
model_instance._gguf_written_hf_names = set()
model_instance._gguf_written_checkpoint_names = set()
@@ -221,15 +228,160 @@ def _quant_data_with_args(
return data
-def _remap_quant_attr_tensor(cls, attr_tensor, module, modify_name, new_name, bid):
- if not isinstance(attr_tensor, torch.Tensor):
- return attr_tensor
- if not (hasattr(cls, "permute") or need_modify_tensor(cls, modify_name)):
- return attr_tensor
+def _tensors_equal(a, b):
+ if a.shape != b.shape:
+ return False
+ if a.dtype != b.dtype:
+ b = b.to(a.dtype)
+ if a.device != b.device:
+ b = b.to(a.device)
+ return torch.equal(a, b)
+
+
+def _trace_modify_transform(cls, in_shape, modify_name, new_name, bid, out_shape):
+ """Probe ``cls.modify_tensors`` with index tensors to identify the weight transform.
+
+ Runs the transform on row-index and column-index tensors (each twice, with an
+ affine pair ``2*x+1`` to reject arithmetic transforms that would fake a valid
+ index map). Returns ``(rowmap, colmap)`` mapping every output row/column back
+ to its source row/column when the transform is a pure gather that keeps rows
+ and columns independent, else None.
+ """
+ rows, cols = in_shape
+
+ def run(probe):
+ outputs = dict(cls.modify_tensors(probe, modify_name, bid))
+ out = outputs.get(new_name)
+ if out is None or tuple(out.shape) != tuple(out_shape):
+ return None
+ return out.to(torch.float32)
+
+ try:
+ row_probe = torch.arange(rows, dtype=torch.float32).unsqueeze(1).expand(rows, cols).contiguous()
+ out_r = run(row_probe)
+ out_r2 = run(row_probe * 2 + 1)
+ del row_probe
+ if out_r is None or out_r2 is None or not torch.equal(out_r2, out_r * 2 + 1):
+ return None # not a pure gather (arithmetic transform)
+ if not torch.equal(out_r, out_r[:, :1].expand_as(out_r)):
+ return None # mixes different source rows into one output row
+ del out_r2
+
+ col_probe = torch.arange(cols, dtype=torch.float32).unsqueeze(0).expand(rows, cols).contiguous()
+ out_c = run(col_probe)
+ out_c2 = run(col_probe * 2 + 1)
+ del col_probe
+ if out_c is None or out_c2 is None or not torch.equal(out_c2, out_c * 2 + 1):
+ return None
+ if not torch.equal(out_c, out_c[:1, :].expand_as(out_c)):
+ return None # mixes different source columns into one output column
+
+ rowmap = out_r[:, 0].round().to(torch.long)
+ colmap = out_c[0].round().to(torch.long)
+ except Exception as e:
+ logger.debug("transform tracing failed for %s: %s", new_name, e)
+ return None
+ if rowmap.numel() == 0 or rowmap.min() < 0 or rowmap.max() >= rows:
+ return None
+ if colmap.numel() == 0 or colmap.min() < 0 or colmap.max() >= cols:
+ return None
+ return rowmap, colmap
+
+
+def _remap_traced_attrs(kwargs, rowmap, colmap, in_shape):
+ """Apply a traced (rowmap, colmap) gather to per-group quant params.
- bs = module.weight.shape[0]
- remapped = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)).get(new_name)
- return attr_tensor if remapped is None else remapped
+ Quant params are row-major with one or more values per group of weight
+ columns, so row gathers always transfer; column gathers transfer only when
+ they move whole param groups. Returns the remapped params, or None when any
+ present param cannot be remapped (params must be used all-or-nothing).
+ """
+ in_rows, in_cols = in_shape
+ col_identity = colmap.numel() == in_cols and torch.equal(colmap, torch.arange(in_cols, dtype=colmap.dtype))
+ remapped = {}
+ for key, t in kwargs.items():
+ if t is None or not isinstance(t, torch.Tensor):
+ remapped[key] = t
+ continue
+ if key == "imatrix":
+ # imatrix is per input column and unaffected by row gathers
+ if col_identity:
+ remapped[key] = t
+ elif t.dim() == 1 and t.numel() == in_cols:
+ remapped[key] = t[colmap.to(t.device)]
+ else:
+ return None
+ continue
+ if t.numel() == 0 or t.numel() % in_rows != 0:
+ return None
+ m = t.reshape(in_rows, -1)
+ m = m.index_select(0, rowmap.to(t.device))
+ if not col_identity:
+ per_row = m.shape[1]
+ if per_row == 0 or in_cols % per_row != 0:
+ return None
+ group_size = in_cols // per_row # weight columns covered by one param value
+ if colmap.numel() % group_size != 0:
+ return None
+ groups = colmap.reshape(-1, group_size)
+ starts = groups[:, 0]
+ expected = starts.unsqueeze(1) + torch.arange(group_size, dtype=colmap.dtype)
+ if not (torch.equal(groups, expected) and bool((starts % group_size == 0).all())):
+ return None # column moves cross param-group boundaries
+ m = m.index_select(1, (starts // group_size).to(t.device))
+ remapped[key] = m
+ return remapped
+
+
+def _adapt_quant_attrs_to_transform(cls, module, data_torch, modify_name, new_name, bid, kwargs):
+ """Make the module's quant params (stored in the original HF weight layout)
+ valid for the tensor produced by ``modify_tensors``.
+
+ Returns the (possibly remapped) params, or None when the transform cannot be
+ applied to them — the caller must then drop the params so ``ggml_quant``
+ re-searches them on the transformed weights.
+ """
+ weight = getattr(module, "weight", None)
+ if weight is None or not isinstance(weight, torch.Tensor):
+ return None
+ if _tensors_equal(data_torch, weight):
+ return kwargs
+ if weight.dim() != 2 or data_torch.dim() != 2:
+ return None
+ trace = _trace_modify_transform(cls, tuple(weight.shape), modify_name, new_name, bid, tuple(data_torch.shape))
+ if trace is None:
+ return None
+ return _remap_traced_attrs(kwargs, trace[0], trace[1], tuple(weight.shape))
+
+
+def _verify_packed_tensor(data, data_qtype, ref, new_name):
+ """Guard against packing with mismatched quant params: dequantize a sample of
+ the packed tensor and compare against the (already fake-quantized) weights.
+ When the provided params match the tensor layout the round-trip is near-exact,
+ so any sizable deviation means the export would be silently corrupted.
+
+ ``ref`` must be a snapshot of the leading weight rows taken BEFORE ggml_quant,
+ whose kernels modify their input buffer in place."""
+ try:
+ arr = data if isinstance(data, np.ndarray) else np.asarray(data)
+ if arr.ndim < 2 or ref.dim() < 2:
+ return
+ k = min(arr.shape[0], ref.shape[0])
+ dequantized = gguf.quants.dequantize(arr[:k], data_qtype)
+ ref = ref[:k].detach().to(torch.float32).cpu().numpy()
+ if dequantized.shape != ref.shape:
+ return
+ rel_err = float(np.linalg.norm(dequantized - ref)) / max(float(np.linalg.norm(ref)), 1e-12)
+ except Exception as e:
+ logger.debug("pack verification skipped for %s: %s", new_name, e)
+ return
+ if rel_err > 0.1:
+ raise ValueError(
+ f"{new_name}: packed tensor deviates from the quantized weights by {rel_err:.1%}; "
+ "the quantization params do not match the tensor layout, the export would be corrupted"
+ )
+ if rel_err > 0.02:
+ logger.warning(f"{new_name}: packed tensor deviates from the quantized weights by {rel_err:.1%}")
def _expected_scale_columns(ggml_type):
@@ -273,13 +425,6 @@ def _validate_quant_attr_shapes(data_torch, data_qtype, kwargs, name, new_name):
)
-def need_modify_tensor(cls, name):
- hf_arch = getattr(cls, "hf_arch", "")
- if hf_arch in "Qwen3NextForCausalLM" and "in_proj_qkvz.weight" in name:
- return True
- return False
-
-
def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=None, use_layer_attrs=True):
"""
@@ -313,48 +458,29 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d
"wmin": module.w_wmin if hasattr(module, "w_wmin") else None,
"imatrix": module.imatrix if hasattr(module, "imatrix") else None,
}
- # patch for Qwen3_5, Qwen3_5 handles some weights specially,
- # but the scale doesn't match; these weights are handled by gguf itself.
- # Define model architectures that need special handling
- QWEN3_5_MODELS = {
- "Qwen3_5ForCausalLM",
- "Qwen3_5MoeForCausalLM",
- "Qwen3_5MoeForConditionalGeneration",
- "Qwen3_5ForConditionalGeneration",
- }
-
- QWEN3_5_SKIP_KEYS = {
- ".in_proj_qkv.",
- ".in_proj_z",
- ".conv1d",
- ".in_proj_b.",
- ".in_proj_a.",
- ".A_log",
- ".dt_bias",
- ".out_proj.",
- }
-
- hf_arch = getattr(cls, "hf_arch", "")
- should_skip = hf_arch in QWEN3_5_MODELS and any(key in name for key in QWEN3_5_SKIP_KEYS)
-
- if use_layer_attrs and not should_skip:
- # support for MOE model with cls experts not linear
- for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:
- if not (hasattr(module, attr) and getattr(module, attr) is not None):
- continue
-
- attr_tensor = getattr(module, attr)
- if not isinstance(attr_tensor, torch.Tensor):
- continue
-
- attr_tensor = _remap_quant_attr_tensor(cls, attr_tensor, module, modify_name, new_name, bid)
-
- # Map attribute names to kwargs keys: w_d_scale -> d_scale, w_d_wmin -> d_wmin, w_wmin -> wmin
- kwargs_key = attr.replace("w_", "") if attr.startswith("w_") else attr
- kwargs[kwargs_key] = attr_tensor.to(torch.float32)
+ if any(isinstance(v, torch.Tensor) for v in kwargs.values()):
+ # modify_tensors may have permuted/split the weights (e.g. llama Q/K
+ # interleave, Qwen3.5 V-head reorder); the stored quant params must
+ # follow the same transform or they describe the wrong rows/columns.
+ adapted = _adapt_quant_attrs_to_transform(cls, module, data_torch, modify_name, new_name, bid, kwargs)
+ if adapted is None:
+ logger.info(
+ f"{new_name}: weight transform is incompatible with the stored quantization params,"
+ " re-searching them on the transformed weights"
+ )
+ adapted = dict.fromkeys(kwargs)
+ kwargs = {
+ k: v.to(torch.float32) if isinstance(v, torch.Tensor) and k != "imatrix" else v
+ for k, v in adapted.items()
+ }
data_torch = data_torch.to(torch.float32)
_validate_quant_attr_shapes(data_torch, data_qtype, kwargs, name, new_name)
+ # ggml_quant kernels modify their input buffer in place; snapshot the rows
+ # used by the post-pack verification first.
+ verify_ref = data_torch[:64].detach().clone() if kwargs.get("scale") is not None and data_torch.dim() >= 2 else None
data = ggml_quant(data_torch, data_qtype.name.lower(), device=device, **kwargs)
+ if verify_ref is not None:
+ _verify_packed_tensor(data, data_qtype, verify_ref, new_name)
# else:
# # if data_torch.dtype ==torch.float32:
# # data_qtype = gguf.GGMLQuantizationType.F32
@@ -437,6 +563,34 @@ def _qtype_name(qtype):
return qtype.name if hasattr(qtype, "name") else str(qtype)
+def _qtype_precision_rank(qtype):
+ ranks = {
+ gguf.GGMLQuantizationType.Q2_K: 2,
+ gguf.GGMLQuantizationType.Q3_K: 3,
+ gguf.GGMLQuantizationType.Q4_0: 4,
+ gguf.GGMLQuantizationType.Q4_1: 4,
+ gguf.GGMLQuantizationType.Q4_K: 4,
+ gguf.GGMLQuantizationType.Q5_0: 5,
+ gguf.GGMLQuantizationType.Q5_1: 5,
+ gguf.GGMLQuantizationType.Q5_K: 5,
+ gguf.GGMLQuantizationType.Q6_K: 6,
+ gguf.GGMLQuantizationType.Q8_0: 8,
+ gguf.GGMLQuantizationType.Q8_1: 8,
+ gguf.GGMLQuantizationType.Q8_K: 8,
+ gguf.GGMLQuantizationType.F16: 16,
+ gguf.GGMLQuantizationType.BF16: 16,
+ gguf.GGMLQuantizationType.F32: 32,
+ }
+ return ranks.get(qtype, 0)
+
+
+def _should_keep_recipe_qtype(layer_config_qtype, fallback_qtype, allow_recipe_fallback):
+ """Keep llama.cpp fixed-format recipe upgrades unless AutoScheme owns dtype selection."""
+ if not allow_recipe_fallback or layer_config_qtype is None:
+ return False
+ return _qtype_precision_rank(fallback_qtype) > _qtype_precision_rank(layer_config_qtype)
+
+
def resolve_restored_qtype(
layer_config,
hf_names,
@@ -444,6 +598,7 @@ def resolve_restored_qtype(
gguf_name,
fallback_qtype,
diagnostics,
+ allow_recipe_fallback=False,
):
source_qtypes = [
get_qtype_by_layer_config(layer_config, hf_name, fallback_qtype, explicit_only=True) for hf_name in hf_names
@@ -451,10 +606,14 @@ def resolve_restored_qtype(
matched_qtypes = [qtype for qtype in source_qtypes if qtype is not None]
if len(matched_qtypes) == 1 and len(hf_names) == 1:
+ if _should_keep_recipe_qtype(matched_qtypes[0], fallback_qtype, allow_recipe_fallback):
+ return None
return matched_qtypes[0]
if len(matched_qtypes) > 0 and len(matched_qtypes) == len(source_qtypes):
unique_qtypes = set(matched_qtypes)
if len(unique_qtypes) == 1:
+ if _should_keep_recipe_qtype(matched_qtypes[0], fallback_qtype, allow_recipe_fallback):
+ return None
return matched_qtypes[0]
if len(hf_names) <= 1 and len(matched_qtypes) == 0:
@@ -747,6 +906,7 @@ def prepare_tensors(cls):
if restored.transform_kind != "extra" and new_name.endswith(".weight") and n_dims > 1
else []
),
+ allow_recipe_fallback=not getattr(cls, "is_auto_scheme", False),
)
# # No override (data_qtype is False), or wants to be quantized (data_qtype is True)
if layer_config_qtype is not None:
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index bf9d36c0a..e5c410cca 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -83,6 +83,7 @@ def create_model_class(
model_type=ModelType.TEXT,
device="cpu",
quant_nontext_module: bool = False,
+ is_auto_scheme: bool = False,
):
tmp_work_dir = model.name_or_path
os.makedirs(output_dir, exist_ok=True)
@@ -134,6 +135,7 @@ def create_model_class(
low_cpu_mem_usage=low_cpu_mem_usage,
device=device,
quant_nontext_module=quant_nontext_module,
+ is_auto_scheme=is_auto_scheme,
)
model_instance = handle_special_model(model_instance, model_architecture)
return model_instance
@@ -152,6 +154,7 @@ def pack_gguf_layer(
model_type=ModelType.TEXT,
device="cpu",
quant_nontext_module=False,
+ is_auto_scheme=False,
):
"""Export the model to gguf format."""
global gguf_model_instance_global
@@ -166,6 +169,7 @@ def pack_gguf_layer(
model_type=ModelType.TEXT,
device=device,
quant_nontext_module=quant_nontext_module,
+ is_auto_scheme=is_auto_scheme,
)
]
if model_type == ModelType.MMPROJ:
@@ -179,6 +183,7 @@ def pack_gguf_layer(
model_type=ModelType.MMPROJ,
device=device,
quant_nontext_module=quant_nontext_module,
+ is_auto_scheme=is_auto_scheme,
)
)
@@ -236,6 +241,7 @@ def save_quantized_as_gguf(
mllm=False,
device="cpu",
quant_nontext_module=False,
+ is_auto_scheme=False,
**kwargs,
):
"""Export the model to gguf format."""
@@ -252,6 +258,7 @@ def save_quantized_as_gguf(
model_type=ModelType.TEXT,
device=device,
quant_nontext_module=quant_nontext_module,
+ is_auto_scheme=is_auto_scheme,
)
]
if mllm:
@@ -264,6 +271,7 @@ def save_quantized_as_gguf(
model_type=ModelType.MMPROJ,
device=device,
quant_nontext_module=quant_nontext_module,
+ is_auto_scheme=is_auto_scheme,
)
)
diff --git a/auto_round/export/export_to_gguf/gguf_dtype.py b/auto_round/export/export_to_gguf/gguf_dtype.py
index 1817ad46d..6c173d3de 100644
--- a/auto_round/export/export_to_gguf/gguf_dtype.py
+++ b/auto_round/export/export_to_gguf/gguf_dtype.py
@@ -223,7 +223,6 @@ class GGUFDTypeSelector:
)
or 0
)
-
# llama.cpp: output & token_embd handling (llama_tensor_get_type outer wrapper)
if category == TensorCategory.OUTPUT or (self.has_tied_embeddings and category == TensorCategory.TOKEN_EMBD):
# llama.cpp: if new_type != Q8_0, upgrade to Q6_K
@@ -266,7 +265,7 @@ class GGUFDTypeSelector:
elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_M:
if i_layer < n_layer // 16:
qtype = gguf.GGMLQuantizationType.Q5_K
- elif self.model_arch == gguf.MODEL_ARCH.FALCON or _use_more_bits(i_layer, n_layer):
+ elif self.model_arch != gguf.MODEL_ARCH.FALCON or _use_more_bits(i_layer, n_layer):
qtype = gguf.GGMLQuantizationType.Q4_K
else:
qtype = gguf.GGMLQuantizationType.Q3_K
diff --git a/auto_round/formats.py b/auto_round/formats.py
index d16de4756..9dd9ce21d 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -783,6 +783,7 @@ class GGUFFormat(OutputFormat):
format_name = "gguf"
def __init__(self, format: str, ar: BaseCompressor):
+ self.is_auto_scheme = bool(getattr(ar, "is_auto_scheme", False))
if format.startswith("gguf:"):
self._original_format = format # preserve "gguf:q2_k_mixed" etc. for Phase 2b
self.output_format = "gguf"
@@ -875,6 +876,7 @@ class GGUFFormat(OutputFormat):
model_type,
device,
quant_nontext_module,
+ is_auto_scheme=self.is_auto_scheme,
)
def save_quantized(
@@ -899,6 +901,7 @@ class GGUFFormat(OutputFormat):
mllm=self.mllm,
device=device,
serialization_dict=serialization_dict,
+ is_auto_scheme=self.is_auto_scheme,
**kwargs,
)
diff --git a/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py
index d06c1f278..3f76bee91 100644
--- a/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py
+++ b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py
@@ -115,7 +115,7 @@ def test_restorer_does_not_rerun_completed_conversion_group(monkeypatch):
assert restored == []
-def test_resolve_restored_qtype_uses_consistent_multi_source_layer_config():
+def test_resolve_restored_qtype_keeps_higher_recipe_fallback_for_multi_source_layer_config():
import gguf
from auto_round.export.export_to_gguf.convert import resolve_restored_qtype
@@ -134,12 +134,60 @@ def test_resolve_restored_qtype_uses_consistent_multi_source_layer_config():
gguf_name="blk.0.ffn_gate_up.weight",
fallback_qtype=gguf.GGMLQuantizationType.Q5_K,
diagnostics=diagnostics,
+ allow_recipe_fallback=True,
+ )
+
+ assert qtype is None
+ assert diagnostics == []
+
+
+def test_resolve_restored_qtype_respects_mixed_layer_config_even_when_recipe_is_higher():
+ import gguf
+
+ from auto_round.export.export_to_gguf.convert import resolve_restored_qtype
+
+ diagnostics = []
+ qtype = resolve_restored_qtype(
+ layer_config={
+ "model.layers.0.mlp.gate_proj": {"bits": 4, "super_bits": 6, "sym": False},
+ "model.layers.0.mlp.up_proj": {"bits": 4, "super_bits": 6, "sym": False},
+ },
+ hf_names=(
+ "model.layers.0.mlp.gate_proj.weight",
+ "model.layers.0.mlp.up_proj.weight",
+ ),
+ checkpoint_name="model.layers.0.mlp.gate_up_proj.weight",
+ gguf_name="blk.0.ffn_gate_up.weight",
+ fallback_qtype=gguf.GGMLQuantizationType.Q5_K,
+ diagnostics=diagnostics,
+ allow_recipe_fallback=False,
)
assert qtype == gguf.GGMLQuantizationType.Q4_K
assert diagnostics == []
+def test_resolve_restored_qtype_keeps_higher_layer_config_than_recipe_fallback():
+ import gguf
+
+ from auto_round.export.export_to_gguf.convert import resolve_restored_qtype
+
+ diagnostics = []
+ qtype = resolve_restored_qtype(
+ layer_config={
+ "model.layers.0.mlp.down_proj": {"bits": 6, "super_bits": 8, "sym": True},
+ },
+ hf_names=("model.layers.0.mlp.down_proj.weight",),
+ checkpoint_name="model.layers.0.mlp.down_proj.weight",
+ gguf_name="blk.0.ffn_down.weight",
+ fallback_qtype=gguf.GGMLQuantizationType.Q5_K,
+ diagnostics=diagnostics,
+ )
+
+ assert qtype == gguf.GGMLQuantizationType.Q6_K
+ assert diagnostics == []
+
+
def test_resolve_restored_qtype_falls_back_and_records_conflicting_multi_source_layer_config():
import gguf
diff --git a/test/test_cpu/export/test_gguf_mtp_dtype.py b/test/test_cpu/export/test_gguf_mtp_dtype.py
index a036b8a0e..6cad988c2 100644
--- a/test/test_cpu/export/test_gguf_mtp_dtype.py
+++ b/test/test_cpu/export/test_gguf_mtp_dtype.py
@@ -1,11 +1,15 @@
+from types import SimpleNamespace
+
import pytest
+from auto_round.compressors.utils import _resolve_gguf_n_layers
from auto_round.export.export_to_gguf.gguf_dtype import (
GGUFDTypeSelector,
gguf_format_to_ftype,
select_llama_cpp_compatible_qtype,
)
from auto_round.utils import LazyImport
+from auto_round.utils.model import ModelType
gguf = LazyImport("gguf")
@@ -115,8 +119,44 @@ def test_attn_v_more_bits_uses_attention_counter_not_layer_id():
assert selector.select_gguf_type("blk.1.attn_v.weight", n_dims=2) == "gguf:q6_k"
+def test_q3_k_m_ffn_down_uses_q4_k_for_non_falcon_like_llama_cpp():
+ # llama.cpp: new_type = i_layer < n_layer/16 ? Q5_K
+ # : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? Q4_K : Q3_K
+ # so for non-Falcon archs every ffn_down beyond the first n_layer/16 layers is Q4_K.
+ hparams = {"num_hidden_layers": 33, "num_attention_heads": 16, "num_key_value_heads": 4}
+ selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q3_k_m"))
+
+ qtypes = {i: selector.select_qtype(f"blk.{i}.ffn_down.weight", n_dims=2) for i in range(33)}
+
+ assert qtypes[0] == qtypes[1] == gguf.GGMLQuantizationType.Q5_K
+ assert all(qtypes[i] == gguf.GGMLQuantizationType.Q4_K for i in range(2, 33))
+
+
def test_q4_0_ffn_down_upgrade_requires_imatrix_like_llama_cpp():
hparams = {"num_hidden_layers": 28, "num_attention_heads": 16, "num_key_value_heads": 4}
selector = GGUFDTypeSelector(hparams, gguf_format_to_ftype("gguf:q4_0"), has_imatrix=False)
assert selector.select_gguf_type("blk.0.ffn_down.weight", n_dims=2) == "gguf:q4_0"
+
+
+def test_n_layer_resolution_reads_nested_text_config():
+ # multimodal configs like Qwen3.5 expose num_hidden_layers only under text_config;
+ # failing to resolve it silently disabled the whole llama.cpp k-quant mixture.
+ config = SimpleNamespace(
+ text_config=SimpleNamespace(num_hidden_layers=32),
+ vision_config=SimpleNamespace(depth=24),
+ )
+
+ assert _resolve_gguf_n_layers(config, ModelType.TEXT) == (32, None)
+ assert _resolve_gguf_n_layers(config, ModelType.MMPROJ) == (32, 24)
+
+
+def test_n_layer_resolution_prefers_text_config_for_multimodal():
+ config = SimpleNamespace(
+ num_hidden_layers=48,
+ text_config=SimpleNamespace(num_hidden_layers=32),
+ vision_config=SimpleNamespace(depth=24),
+ )
+
+ assert _resolve_gguf_n_layers(config, ModelType.MMPROJ) == (32, 24)
+ assert _resolve_gguf_n_layers(config, ModelType.TEXT) == (48, None)
diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.py
index 2c76f0671..458266b86 100644
--- a/test/test_cuda/export/test_gguf_format.py
+++ b/test/test_cuda/export/test_gguf_format.py
@@ -175,7 +175,7 @@ class TestAutoRound:
print(f"{file}: {os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2} MB")
file_size = os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2
if "mmproj-model.gguf" in file:
- assert abs(file_size - 62) < 5.0
+ assert abs(file_size - 75) < 5.0
else:
assert abs(file_size - 690) < 10.0