Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix few gguf bugs of hf-name and checkpoint name (#1939)

Heng Guo126793d50cf8
diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.py
index 23d11518e..1770b88b4 100644
--- a/auto_round/compressors/entry.py
+++ b/auto_round/compressors/entry.py
@@ -362,7 +362,8 @@ class AutoRound(object):
# Model-free routing is now supported directly by the new entry path.
model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)
model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)
- if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_kwargs):
+ route_decision_kwargs = dict(route_kwargs, format=format)
+ if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs):
from auto_round.compressors.model_free import ModelFreeCompressor
if not isinstance(model, str):
diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.py
index b94a81fab..5d2f33453 100644
--- a/auto_round/compressors/model_free.py
+++ b/auto_round/compressors/model_free.py
@@ -1948,6 +1948,11 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):
# Start from the remaining user kwargs and explicitly set/override
# known compressor init parameters for clarity.
fallback_init = dict(fallback_kwargs)
+ # Route-control kwargs are only meaningful for the initial entry
+ # selection. Strip them so fallback always re-enters the regular flow
+ # with a single explicit disable_model_free=True override.
+ fallback_init.pop("model_free", None)
+ fallback_init.pop("disable_model_free", None)
fallback_init.update(
model=model_name_or_path,
iters=0,
diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.py
index a002fbfb2..74d851aca 100644
--- a/auto_round/compressors/utils.py
+++ b/auto_round/compressors/utils.py
@@ -611,6 +611,7 @@ def set_layer_config(
model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXT
layer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)
+ _apply_gguf_shape_fallback(layer_config, model)
dispatch_layer_config(layer_config)
return layer_config, has_qlayer_outside_block, regex_config
@@ -681,6 +682,48 @@ def get_gguf_qtype_by_layer_config(layer_config):
raise ValueError("Unknown layer config")
+def _apply_gguf_shape_fallback(layer_config, model):
+ from auto_round.utils.model import get_module
+
+ for layer_name, config in layer_config.items():
+ if not check_to_quantized(config):
+ continue
+ layer = get_module(model, layer_name)
+ if layer is None or not hasattr(layer, "weight"):
+ continue
+ try:
+ qtype = get_gguf_qtype_by_layer_config(config)
+ except ValueError:
+ continue
+ if qtype is None:
+ continue
+
+ gguf_type = f"gguf:{qtype.name.lower()}"
+ block_size = GGML_QUANT_SIZES[gguf_type.split(":")[-1]][0]
+ input_features = (
+ layer.weight.shape[0] if type(layer) == transformers.pytorch_utils.Conv1D else layer.weight.shape[-1]
+ )
+ if input_features % block_size == 0:
+ continue
+
+ fallback_type = _gguf_type_fallback(gguf_type)
+ fallback_block_size = GGML_QUANT_SIZES[fallback_type.split(":")[-1]][0]
+ if input_features % fallback_block_size != 0:
+ fallback_type = "gguf:bf16"
+
+ preserved = {key: config[key] for key in ("fixed_by_user", "scale_dtype") if key in config}
+ config.update(GGUF_INNER_CONFIG[fallback_type])
+ config.update(preserved)
+ logger.warning(
+ "fallback %s to %s before quantization, because input_features(%s) is not divisible by %s block_size(%s)",
+ layer_name,
+ fallback_type,
+ input_features,
+ gguf_type,
+ block_size,
+ )
+
+
def _get_digital_in_layer_name(layer_name):
pattern = re.compile(r"([a-zA-Z]+\.){1,}(\d+)")
res = re.search(pattern, layer_name)
diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.py
index be9baadfb..f6a4bd2a9 100644
--- a/auto_round/eval/evaluation.py
+++ b/auto_round/eval/evaluation.py
@@ -184,6 +184,28 @@ def evaluate_diffusion_model(args, autoround=None, model=None, pipe=None):
diffusion_eval(pipe, args.prompt_file, metrics, args.image_save_dir, 1, gen_kwargs, args.limit)
+def select_gguf_eval_file(eval_folder, formats):
+ """Select the text GGUF file used for HF-based evaluation."""
+ gguf_format = None
+ for format in formats:
+ if format.startswith("gguf"):
+ gguf_format = format.split(":")[-1].upper()
+ break
+
+ if gguf_format is None:
+ return None, []
+
+ gguf_files = sorted(
+ file for file in os.listdir(eval_folder) if file.endswith(".gguf") and file != "mmproj-model.gguf"
+ )
+ matched_files = [file for file in gguf_files if gguf_format in file]
+ if matched_files:
+ return matched_files[0], gguf_files
+ if len(gguf_files) == 1:
+ return gguf_files[0], gguf_files
+ return None, gguf_files
+
+
def load_gguf_model_for_eval(eval_folder, formats, args):
"""
Load GGUF model for evaluation.
@@ -202,26 +224,13 @@ def load_gguf_model_for_eval(eval_folder, formats, args):
from auto_round.utils import get_model_dtype, logger
- # Find corresponding GGUF format
- gguf_format = None
- for format in formats:
- if format.startswith("gguf"):
- gguf_format = format.split(":")[-1].upper()
- break
-
- if gguf_format is None:
+ gguf_file, gguf_files = select_gguf_eval_file(eval_folder, formats)
+ if not any(format.startswith("gguf") for format in formats):
logger.error("No valid gguf format found in formats. Please check the input.")
sys.exit(-1)
- # Find matching GGUF file
- gguf_file = None
- for file in os.listdir(eval_folder):
- if gguf_format in file:
- gguf_file = file
- break
-
if gguf_file is None:
- logger.error("Cannot find correct gguf file for evaluation, please check.")
+ logger.error("Cannot find correct gguf file for evaluation, candidates=%s", gguf_files)
sys.exit(-1)
# Load model and tokenizer
@@ -482,6 +491,8 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):
# Load or prepare model instance
if eval_gguf_model:
model, tokenizer = load_gguf_model_for_eval(eval_folder, formats, args)
+ if model is None:
+ return
else:
eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")
model = prepare_model_for_eval(model, args.device_map, eval_model_dtype)
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 61ac14bb5..be21e33e4 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -39,7 +39,6 @@ import os
import re
from functools import partial
from itertools import chain
-from pathlib import Path
from typing import TYPE_CHECKING, Any, Iterator
import numpy as np
@@ -48,6 +47,7 @@ from transformers import AutoConfig
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector
+from auto_round.export.export_to_gguf.hf_checkpoint_restorer import HFCheckpointRestorer, RestoredTensor
from auto_round.export.export_to_gguf.packing import ggml_quant
from auto_round.utils import (
LazyImport,
@@ -75,8 +75,10 @@ def wrapper_model_instance(
model_instance.model = model
model_instance.layer_config = layer_config
model_instance.low_cpu_mem_usage = _need_low_cpu_mem(low_cpu_mem_usage)
+ model_instance._gguf_written_hf_names = set()
+ model_instance._gguf_written_checkpoint_names = set()
- model_instance.get_tensors = partial(get_tensors, model_instance)
+ model_instance.get_tensors = partial(get_restored_tensors, model_instance)
model_instance.prepare_tensors = partial(prepare_tensors, model_instance)
model_instance.device = device
@@ -130,30 +132,11 @@ def is_mmproj_tensor_name(name):
return any(key in name for key in MM_KEYS)
-def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
- if not hasattr(cls.model, "tensor_name_list"):
- cls.model.tensor_name_list = []
-
- checkpoint_conversion_mapping = getattr(cls.model, "_checkpoint_conversion_mapping", {}) or {}
- reverse_key_mapping = {v: k for k, v in checkpoint_conversion_mapping.items()}
- for name, tensor in cls.model.named_parameters():
- if name not in cls.model.tensor_name_list:
- cls.model.tensor_name_list.append(name)
- for pattern, replacement in reverse_key_mapping.items():
- replacement = replacement.lstrip("^") # strip off un-needed chars and patterns
- replacement = re.sub(r"\(.*\)", "", replacement)
- key, n_replace = re.subn(pattern, replacement, name)
- cls.model.tensor_name_list.append(key)
- if n_replace > 0:
- break
- yield name, tensor
-
+def _iter_extra_tensors(cls):
def is_extra_tensor(tensor_name):
if getattr(cls.model, "_is_quantized_input_module", False) and "scale" in tensor_name.split(".")[-1]:
return False
- if tensor_name not in cls.model.tensor_name_list:
- return True
- return False
+ return tensor_name not in getattr(cls.model, "tensor_name_list", [])
extra_tensor = {}
if hasattr(cls.model, "name_or_path"):
@@ -173,13 +156,43 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
if is_extra_tensor(tensor_name):
extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)
else:
- f = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")
- for tensor_name in f.keys():
- if is_extra_tensor(tensor_name):
- extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)
-
- for name, tensor in extra_tensor.items():
- yield name, tensor
+ model_file = os.path.join(dir_path, "model.safetensors")
+ if os.path.exists(model_file):
+ f = safe_open(model_file, framework="pt")
+ for tensor_name in f.keys():
+ if is_extra_tensor(tensor_name):
+ extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)
+
+ yield from extra_tensor.items()
+
+
+def get_restored_tensors(cls) -> Iterator[RestoredTensor]:
+ written_hf_names = getattr(cls, "_gguf_written_hf_names", set())
+ written_checkpoint_names = getattr(cls, "_gguf_written_checkpoint_names", set())
+ cls.model.tensor_name_list = list(written_hf_names | written_checkpoint_names)
+
+ pending_checkpoint_tensors = {}
+ for restored in HFCheckpointRestorer(cls.model, completed_hf_names=written_hf_names).iter_tensors():
+ tensor = restored.tensor_fn()
+ if tensor is None or tensor.numel() == 0:
+ pending_checkpoint_tensors[restored.checkpoint_name] = restored
+ continue
+ for tensor_name in (restored.checkpoint_name, *restored.hf_names):
+ if tensor_name not in cls.model.tensor_name_list:
+ cls.model.tensor_name_list.append(tensor_name)
+ yield RestoredTensor(
+ restored.checkpoint_name,
+ lambda tensor=tensor: tensor,
+ restored.hf_names,
+ restored.transform_kind,
+ )
+
+ for name, tensor in _iter_extra_tensors(cls):
+ pending = pending_checkpoint_tensors.pop(name, None)
+ if pending is None:
+ yield RestoredTensor(name, lambda tensor=tensor: tensor, (name,), "extra")
+ else:
+ yield RestoredTensor(name, lambda tensor=tensor: tensor, pending.hf_names, pending.transform_kind)
def _quant_data_with_args(
@@ -208,6 +221,58 @@ def _quant_data_with_args(
return data
+def _remap_quant_attr_tensor(cls, attr_tensor, module, modify_name, new_name, bid):
+ if not isinstance(attr_tensor, torch.Tensor):
+ return attr_tensor
+ if not (hasattr(cls, "permute") or need_modify_tensor(cls, modify_name)):
+ return attr_tensor
+
+ bs = module.weight.shape[0]
+ remapped = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)).get(new_name)
+ return attr_tensor if remapped is None else remapped
+
+
+def _expected_scale_columns(ggml_type):
+ from auto_round.export.export_to_gguf.config import QK_K
+
+ if ggml_type in {"q4_0", "q4_1", "q5_0", "q5_1", "q8_0"}:
+ return 1
+ if ggml_type in {"q2_k", "q3_k", "q6_k"}:
+ return QK_K // 16
+ if ggml_type in {"q4_k", "q5_k"}:
+ return QK_K // 32
+ return None
+
+
+def _validate_quant_attr_shapes(data_torch, data_qtype, kwargs, name, new_name):
+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES
+
+ ggml_type = data_qtype.name.lower()
+ if ggml_type not in GGML_QUANT_SIZES or kwargs.get("scale") is None:
+ return
+
+ block_size, _ = GGML_QUANT_SIZES[ggml_type]
+ n_blocks = data_torch.numel() // block_size
+ scale_cols = _expected_scale_columns(ggml_type)
+ if scale_cols is None:
+ return
+
+ scale = kwargs["scale"]
+ if scale.numel() != n_blocks * scale_cols:
+ raise ValueError(
+ f"{name} -> {new_name}: {ggml_type} scale shape {tuple(scale.shape)} does not match "
+ f"tensor shape {tuple(data_torch.shape)}; expected {n_blocks * scale_cols} scale values"
+ )
+
+ for attr in ["d_scale", "d_wmin"]:
+ tensor = kwargs.get(attr)
+ if tensor is not None and tensor.numel() != n_blocks:
+ raise ValueError(
+ f"{name} -> {new_name}: {ggml_type} {attr} shape {tuple(tensor.shape)} does not match "
+ f"tensor shape {tuple(data_torch.shape)}; expected {n_blocks} values"
+ )
+
+
def need_modify_tensor(cls, name):
hf_arch = getattr(cls, "hf_arch", "")
if hf_arch in "Qwen3NextForCausalLM" and "in_proj_qkvz.weight" in name:
@@ -215,7 +280,7 @@ def need_modify_tensor(cls, name):
return False
-def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=None):
+def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=None, use_layer_attrs=True):
"""
Args:
@@ -236,14 +301,18 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d
else:
layer_name = name
module = get_module(cls.model, layer_name)
- kwargs = {
- "scale": module.scale if hasattr(module, "scale") else None,
- "zp": module.zp if hasattr(module, "zp") else None,
- "d_scale": module.w_d_scale if hasattr(module, "w_d_scale") else None,
- "d_wmin": module.w_d_wmin if hasattr(module, "w_d_wmin") else None,
- "wmin": module.w_wmin if hasattr(module, "w_wmin") else None,
- "imatrix": module.imatrix if hasattr(module, "imatrix") else None,
- }
+ kwargs = {"scale": None, "zp": None, "d_scale": None, "d_wmin": None, "wmin": None, "imatrix": None}
+ source_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype, explicit_only=True)
+ use_layer_attrs = use_layer_attrs and source_qtype == data_qtype
+ if use_layer_attrs:
+ kwargs = {
+ "scale": module.scale if hasattr(module, "scale") else None,
+ "zp": module.zp if hasattr(module, "zp") else None,
+ "d_scale": module.w_d_scale if hasattr(module, "w_d_scale") else None,
+ "d_wmin": module.w_d_wmin if hasattr(module, "w_d_wmin") else None,
+ "wmin": module.w_wmin if hasattr(module, "w_wmin") else None,
+ "imatrix": module.imatrix if hasattr(module, "imatrix") else None,
+ }
# patch for Qwen3_5, Qwen3_5 handles some weights specially,
# but the scale doesn't match; these weights are handled by gguf itself.
# Define model architectures that need special handling
@@ -268,7 +337,7 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d
hf_arch = getattr(cls, "hf_arch", "")
should_skip = hf_arch in QWEN3_5_MODELS and any(key in name for key in QWEN3_5_SKIP_KEYS)
- if not should_skip:
+ if use_layer_attrs and not should_skip:
# support for MOE model with cls experts not linear
for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:
if not (hasattr(module, attr) and getattr(module, attr) is not None):
@@ -278,15 +347,13 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, d
if not isinstance(attr_tensor, torch.Tensor):
continue
- if hasattr(cls, "permute") or need_modify_tensor(cls, name):
- bs = module.weight.shape[0]
- attr_tensors_dict = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid))
- attr_tensor = attr_tensors_dict[new_name]
+ attr_tensor = _remap_quant_attr_tensor(cls, attr_tensor, module, modify_name, new_name, bid)
# Map attribute names to kwargs keys: w_d_scale -> d_scale, w_d_wmin -> d_wmin, w_wmin -> wmin
kwargs_key = attr.replace("w_", "") if attr.startswith("w_") else attr
kwargs[kwargs_key] = attr_tensor.to(torch.float32)
data_torch = data_torch.to(torch.float32)
+ _validate_quant_attr_shapes(data_torch, data_qtype, kwargs, name, new_name)
data = ggml_quant(data_torch, data_qtype.name.lower(), device=device, **kwargs)
# else:
# # if data_torch.dtype ==torch.float32:
@@ -366,6 +433,57 @@ def get_qtype_by_layer_config(layer_config, name, data_qtype, *, explicit_only=F
# return data_qtype
+def _qtype_name(qtype):
+ return qtype.name if hasattr(qtype, "name") else str(qtype)
+
+
+def resolve_restored_qtype(
+ layer_config,
+ hf_names,
+ checkpoint_name,
+ gguf_name,
+ fallback_qtype,
+ diagnostics,
+):
+ source_qtypes = [
+ get_qtype_by_layer_config(layer_config, hf_name, fallback_qtype, explicit_only=True) for hf_name in hf_names
+ ]
+ matched_qtypes = [qtype for qtype in source_qtypes if qtype is not None]
+
+ if len(matched_qtypes) == 1 and len(hf_names) == 1:
+ return matched_qtypes[0]
+ if len(matched_qtypes) > 0 and len(matched_qtypes) == len(source_qtypes):
+ unique_qtypes = set(matched_qtypes)
+ if len(unique_qtypes) == 1:
+ return matched_qtypes[0]
+
+ if len(hf_names) <= 1 and len(matched_qtypes) == 0:
+ reason = "layer_config_miss"
+ elif len(matched_qtypes) == 0:
+ reason = "multi_source_layer_config_miss"
+ elif len(matched_qtypes) != len(source_qtypes):
+ reason = "multi_source_partial_layer_config_miss"
+ else:
+ reason = "multi_source_dtype_conflict"
+
+ diagnostics.append(
+ {
+ "checkpoint_name": checkpoint_name,
+ "gguf_name": gguf_name,
+ "reason": reason,
+ "fallback_qtype": _qtype_name(fallback_qtype),
+ "hf_sources": [
+ {
+ "name": hf_name,
+ "layer_config_qtype": None if qtype is None else _qtype_name(qtype),
+ }
+ for hf_name, qtype in zip(hf_names, source_qtypes)
+ ],
+ }
+ )
+ return None
+
+
def _special_name_handle(cls, name):
# using transformers >= 5.4, model.language_model.embed_tokens.weight changed to
# model.language_model.model.embed_tokens.weight after saved
@@ -416,15 +534,45 @@ def _special_name_handle(cls, name):
return name
+def _to_restored_tensor(item):
+ if isinstance(item, RestoredTensor):
+ return item
+ name, data_torch = item
+ return RestoredTensor(name, lambda data_torch=data_torch: data_torch, (name,), "passthrough")
+
+
+def filter_restored_tensor(cls, restored):
+ filtered = cls.filter_tensors((restored.checkpoint_name, restored.tensor_fn))
+ if filtered is None:
+ return None
+ return RestoredTensor(filtered[0], filtered[1], restored.hf_names, restored.transform_kind)
+
+
+def _gguf_writer_has_tensor(gguf_writer, tensor_name):
+ for tensor_info in gguf_writer.tensors:
+ if isinstance(tensor_info, dict):
+ if tensor_name in tensor_info:
+ return True
+ continue
+ existing_name = getattr(tensor_info, "name", None)
+ if existing_name is None and isinstance(tensor_info, (list, tuple)) and len(tensor_info) > 0:
+ existing_name = tensor_info[0]
+ if tensor_name == existing_name:
+ return True
+ return False
+
+
def _count_attention_wv_tensors(cls):
count = 0
- for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):
- if data_torch is None or data_torch.numel() == 0:
- continue
- filtered = cls.filter_tensors((name, lambda data_torch=data_torch: data_torch))
+ for item in chain(cls.generate_extra_tensors(), cls.get_tensors()):
+ restored = _to_restored_tensor(item)
+ filtered = filter_restored_tensor(cls, restored)
if filtered is None:
continue
- modify_name = _special_name_handle(cls, filtered[0])
+ data_torch = filtered.tensor_fn()
+ if data_torch is None or data_torch.numel() == 0:
+ continue
+ modify_name = _special_name_handle(cls, filtered.checkpoint_name)
try:
new_name = cls.map_tensor_name(modify_name)
except Exception:
@@ -433,11 +581,20 @@ def _count_attention_wv_tensors(cls):
return count or None
+def _flush_name_resolution_diagnostics(cls):
+ diagnostics = getattr(cls, "_gguf_name_resolution_diagnostics", None)
+ if diagnostics is None or len(diagnostics) == 0:
+ return
+ logger.debug("gguf: recorded %d in-memory name resolution fallback diagnostics", len(diagnostics))
+
+
def prepare_tensors(cls):
device = get_packing_device(cls.device)
if not hasattr(cls, "_gguf_dtype_selector"):
cls._gguf_dtype_selector = GGUFDTypeSelector(cls.hparams, cls.ftype, cls.model_arch)
cls._gguf_dtype_selector.n_attention_wv = _count_attention_wv_tensors(cls)
+ if not hasattr(cls, "_gguf_name_resolution_diagnostics"):
+ cls._gguf_name_resolution_diagnostics = []
# Handle empty tensor_map for models with block_count=0 (like MobileNetV5)
if cls.tensor_map.mapping:
@@ -445,12 +602,18 @@ def prepare_tensors(cls):
else:
max_name_len = len("vision_encoder.weight,") # Default reasonable length
- for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):
+ for item in chain(cls.generate_extra_tensors(), cls.get_tensors()):
+ restored = _to_restored_tensor(item)
+ name = restored.checkpoint_name
is_mmproj_model = cls.model_arch == gguf.MODEL_ARCH.MMPROJ
if is_mmproj_model != is_mmproj_tensor_name(name):
continue
- if name in getattr(cls.model, "_tied_weights_keys", []) and not is_separate_tensor(cls.model, name):
+ if any(
+ hf_name in getattr(cls.model, "_tied_weights_keys", []) and not is_separate_tensor(cls.model, hf_name)
+ for hf_name in restored.hf_names
+ ):
continue
+ data_torch = restored.tensor_fn()
if data_torch is None or data_torch.numel() == 0:
continue
# we don't need these
@@ -465,10 +628,12 @@ def prepare_tensors(cls):
):
continue
- filtered = cls.filter_tensors((name, lambda data_torch=data_torch: data_torch))
+ filtered = filter_restored_tensor(cls, restored)
if filtered is None:
continue
- name = filtered[0]
+ checkpoint_name = filtered.checkpoint_name
+ hf_names = filtered.hf_names or (checkpoint_name,)
+ hf_name = hf_names[0]
old_dtype = data_torch.dtype
@@ -485,20 +650,19 @@ def prepare_tensors(cls):
clean_weight_list = []
- modify_name = _special_name_handle(cls, name)
+ modify_name = _special_name_handle(cls, checkpoint_name)
+ restored_outputs_completed = False
for new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):
- skip = False
- for tensor_info in cls.gguf_writer.tensors:
- if new_name in tensor_info:
- logger.info(f"{new_name} already add to gguf_writer, skip")
- skip = True
- break
- if skip:
+ restored_outputs_completed = True
+ if _gguf_writer_has_tensor(cls.gguf_writer, new_name):
+ logger.debug("%s already added to gguf_writer, skip", new_name)
continue
# squeeze is necessary for reloading in transformers.
data = data_torch.squeeze()
n_dims = len(data.shape)
- data_qtype: gguf.GGMLQuantizationType | bool = cls.tensor_force_quant(name, new_name, bid, n_dims)
+ data_qtype: gguf.GGMLQuantizationType | bool = cls.tensor_force_quant(
+ checkpoint_name, new_name, bid, n_dims
+ )
# Most of the codebase that takes in 1D tensors or norms only handles F32 tensors
if n_dims <= 1 or new_name.endswith("_norm.weight"):
@@ -557,16 +721,36 @@ def prepare_tensors(cls):
data_qtype = gguf.GGMLQuantizationType.F16
# data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type
- # get name by new_name (for experts),
- name = get_moe_name(cls, name, new_name)
- clean_weight_list.append(name)
- # get data_qtype by layer_config
- layer_config_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype, explicit_only=True)
- # # No override (data_qtype is False), or wants to be quantized (data_qtype is True)
- if layer_config_qtype is not None:
- data_qtype = layer_config_qtype
- elif isinstance(data_qtype, bool):
- data_qtype = cls._gguf_dtype_selector.select_qtype(new_name, n_dims)
+ if cls.model_arch == gguf.MODEL_ARCH.MMPROJ and cls.quant_nontext_module is False:
+ data_qtype = gguf.GGMLQuantizationType.F32
+ layer_config_names = hf_names
+ else:
+ # get name by new_name (for experts),
+ layer_config_names = tuple(get_moe_name(cls, source_name, new_name) for source_name in hf_names)
+ # get data_qtype by layer_config
+ fallback_qtype = (
+ cls._gguf_dtype_selector.select_qtype(new_name, n_dims)
+ if isinstance(data_qtype, bool)
+ else data_qtype
+ )
+ layer_config_qtype = resolve_restored_qtype(
+ cls.layer_config,
+ layer_config_names,
+ checkpoint_name,
+ new_name,
+ fallback_qtype,
+ (
+ cls._gguf_name_resolution_diagnostics
+ if restored.transform_kind != "extra" and new_name.endswith(".weight") and n_dims > 1
+ else []
+ ),
+ )
+ # # No override (data_qtype is False), or wants to be quantized (data_qtype is True)
+ if layer_config_qtype is not None:
+ data_qtype = layer_config_qtype
+ elif isinstance(data_qtype, bool):
+ data_qtype = fallback_qtype
+ clean_weight_list.extend(layer_config_names)
if isinstance(data_qtype, bool):
if cls.ftype == gguf.LlamaFileType.ALL_F32:
@@ -614,9 +798,6 @@ def prepare_tensors(cls):
elif data_qtype == gguf.GGMLQuantizationType.Q6_K:
data_qtype = gguf.GGMLQuantizationType.Q8_0
- if cls.model_arch == gguf.MODEL_ARCH.MMPROJ and cls.quant_nontext_module is False:
- data_qtype = gguf.GGMLQuantizationType.F32
-
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES
if data_qtype.name.lower() in GGML_QUANT_SIZES:
@@ -628,6 +809,16 @@ def prepare_tensors(cls):
)
data_qtype = gguf.GGMLQuantizationType.F16
+ if n_dims > 1 and data_torch.numel() >= 100_000_000:
+ pre_quant_shape = f"{{{', '.join(str(n) for n in reversed(data_torch.shape))}}}"
+ logger.info(
+ "Start quantizing large tensor %s: %s --> %s, shape = %s",
+ new_name,
+ old_dtype,
+ data_qtype.name,
+ pre_quant_shape,
+ )
+
if isinstance(data_qtype, bool) or data_qtype in [
gguf.GGMLQuantizationType.F16,
gguf.GGMLQuantizationType.BF16,
@@ -647,8 +838,8 @@ def prepare_tensors(cls):
data = gguf.quants.quantize(data, data_qtype)
else:
# for deepseek v2
- if name.endswith("kv_b_proj.weight") and cls.model_arch.name in ("DEEPSEEK2", "GLM_DSA"):
- layer_name = name[: -len(".weight")]
+ if hf_name.endswith("kv_b_proj.weight") and cls.model_arch.name in ("DEEPSEEK2", "GLM_DSA"):
+ layer_name = hf_name[: -len(".weight")]
module = get_module(cls.model, layer_name)
n_head_kv = cls.hparams["num_key_value_heads"]
v_head_dim = cls.hparams["v_head_dim"]
@@ -676,10 +867,10 @@ def prepare_tensors(cls):
data = _quant_data_with_args(data_torch, data_qtype, device=device, **attr_list)
# for MOE model
- elif len(data_torch.shape) == 3 and len(re.findall(r"\d+", name)) == 2:
+ elif len(data_torch.shape) == 3 and len(re.findall(r"\d+", hf_name)) == 2:
new_data = []
for idx, arr in enumerate(data_torch):
- arr_name = name.split(".")
+ arr_name = hf_name.split(".")
for i in range(len(arr_name) - 1, -1, -1):
if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):
arr_name[i] = str(idx)
@@ -692,7 +883,15 @@ def prepare_tensors(cls):
del new_data
else:
data, data_qtype = _quant_data(
- cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=device
+ cls,
+ data_torch,
+ data_qtype,
+ hf_name,
+ modify_name,
+ new_name,
+ bid,
+ device=device,
+ use_layer_attrs=len(hf_names) == 1,
)
shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape
@@ -709,6 +908,10 @@ def prepare_tensors(cls):
cls.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)
+ if restored_outputs_completed:
+ cls._gguf_written_checkpoint_names.add(restored.checkpoint_name)
+ cls._gguf_written_hf_names.update(restored.hf_names)
+
# # save cpu memory, but slow
if cls.low_cpu_mem_usage:
for weight_name in clean_weight_list:
@@ -723,3 +926,5 @@ def prepare_tensors(cls):
if cls.model_arch == gguf.MODEL_ARCH.LLAMA and "embed_tokens.weight" in weight_name:
continue
clean_module_parameter(module, weight_name.split(".")[-1])
+ if not (hasattr(cls, "current_packing_block") and cls.current_packing_block is not None):
+ _flush_name_resolution_diagnostics(cls)
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index ecc776786..bf9d36c0a 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -64,6 +64,16 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
}
+def _use_native_nontext_gguf_export(model_type, quant_nontext_module: bool) -> bool:
+ return model_type == ModelType.MMPROJ and not quant_nontext_module
+
+
+def _set_mmproj_output_path(model_instance):
+ if model_instance.fname_out.is_dir():
+ model_instance.fname_out = model_instance.fname_out / "mmproj-model.gguf"
+ return model_instance
+
+
def create_model_class(
output_dir,
model,
@@ -94,7 +104,8 @@ def create_model_class(
else:
model_name = model_name[-1]
- output_type = backend.split(":")[-1]
+ native_nontext_export = _use_native_nontext_gguf_export(model_type, quant_nontext_module)
+ output_type = "f32" if native_nontext_export else backend.split(":")[-1]
if output_type.lower() not in FTYPE_MAP:
raise TypeError(f"{output_type} type is not supported")
output_type = FTYPE_MAP.get(output_type.lower())
@@ -112,14 +123,18 @@ def create_model_class(
dry_run=False,
small_first_shard=False,
)
- model_instance = wrapper_model_instance(
- model_instance,
- model=model,
- layer_config=layer_config,
- low_cpu_mem_usage=low_cpu_mem_usage,
- device=device,
- quant_nontext_module=quant_nontext_module,
- )
+ if native_nontext_export:
+ logger.info("Using native llama.cpp F32 export for non-text GGUF model")
+ model_instance = _set_mmproj_output_path(model_instance)
+ else:
+ model_instance = wrapper_model_instance(
+ model_instance,
+ model=model,
+ layer_config=layer_config,
+ low_cpu_mem_usage=low_cpu_mem_usage,
+ device=device,
+ quant_nontext_module=quant_nontext_module,
+ )
model_instance = handle_special_model(model_instance, model_architecture)
return model_instance
@@ -253,6 +268,8 @@ def save_quantized_as_gguf(
)
for gguf_model in gguf_model_instance_global:
+ model_kind = "mmproj" if gguf_model.model_arch == gguf.MODEL_ARCH.MMPROJ else "text"
+ logger.info("Start writing %s GGUF model to %s", model_kind, gguf_model.fname_out)
gguf_model.write()
rt = time.time() - st
logger.info(f"Model successfully exported to {gguf_model.fname_out}, running time={rt}")
diff --git a/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py b/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py
new file mode 100644
index 000000000..8f6b8f14e
--- /dev/null
+++ b/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py
@@ -0,0 +1,203 @@
+# Copyright (c) 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from collections import defaultdict
+from copy import deepcopy
+from dataclasses import dataclass
+from typing import Callable, Iterator
+
+import torch
+
+from auto_round.utils import logger
+
+
+@dataclass(frozen=True)
+class RestoredTensor:
+ checkpoint_name: str
+ tensor_fn: Callable[[], torch.Tensor]
+ hf_names: tuple[str, ...]
+ transform_kind: str
+
+
+class HFCheckpointRestorer:
+ """Expose a Transformers model state dict as checkpoint-style tensors.
+
+ Transformers may rename or structurally convert checkpoint tensors while loading
+ a model. GGUF conversion expects the original checkpoint namespace, while
+ AutoRound layer config is keyed by live HF module names. This restorer keeps
+ both names available.
+ """
+
+ _SUPPORTED_OPS = {"Chunk", "Concatenate", "MergeModulelist", "SplitModulelist"}
+
+ def __init__(self, model, completed_hf_names=None):
+ self.model = model
+ self.completed_hf_names = completed_hf_names or set()
+
+ def iter_tensors(self) -> Iterator[RestoredTensor]:
+ state_dict = self.model.state_dict()
+ weight_conversions = self._get_weight_conversions()
+ if not weight_conversions:
+ for name, tensor in state_dict.items():
+ if name in self.completed_hf_names:
+ continue
+ yield RestoredTensor(name, lambda tensor=tensor: tensor, (name,), "passthrough")
+ return
+
+ inverted_transforms = self._get_inverted_transforms(weight_conversions)
+ inverted_converters = [transform for transform in inverted_transforms if self._is_weight_converter(transform)]
+ inverted_renamings = [
+ transform for transform in inverted_transforms if not self._is_weight_converter(transform)
+ ]
+ pattern_to_converter = {
+ pattern: converter for converter in inverted_converters for pattern in converter.source_patterns
+ }
+
+ conversion_mapping = {}
+ converter_lineage = defaultdict(list)
+ converter_fallback_tensors = defaultdict(list)
+ converter_order = {}
+ ordered_tensors = []
+
+ for source_order, (original_key, tensor) in enumerate(state_dict.items()):
+ converter_key, matched_pattern = self._rename_source_key(original_key, [], inverted_converters)
+ checkpoint_key, _ = self._rename_source_key(converter_key, inverted_renamings, [])
+
+ if matched_pattern is None:
+ if original_key in self.completed_hf_names:
+ continue
+ transform_kind = "rename" if checkpoint_key != original_key else "passthrough"
+ ordered_tensors.append(
+ (
+ source_order,
+ RestoredTensor(
+ checkpoint_key,
+ lambda tensor=tensor: tensor,
+ (original_key,),
+ transform_kind,
+ ),
+ )
+ )
+ continue
+
+ mapping = conversion_mapping.setdefault(converter_key, deepcopy(pattern_to_converter[matched_pattern]))
+ mapping.add_tensor(checkpoint_key, original_key, matched_pattern, tensor)
+ converter_lineage[converter_key].append(original_key)
+ converter_order.setdefault(converter_key, source_order)
+ converter_fallback_tensors[converter_key].append(
+ RestoredTensor(checkpoint_key, lambda tensor=tensor: tensor, (original_key,), "passthrough")
+ )
+
+ for layer_name, mapping in sorted(conversion_mapping.items()):
+ if all(name in self.completed_hf_names for name in converter_lineage[layer_name]):
+ continue
+ try:
+ realized = mapping.convert(layer_name, model=self.model, config=getattr(self.model, "config", None))
+ except Exception as exc:
+ logger.debug(
+ "gguf: falling back to original tensors for failed Transformers weight conversion %s: %s",
+ layer_name,
+ exc,
+ )
+ ordered_tensors.extend(
+ (converter_order[layer_name], tensor) for tensor in converter_fallback_tensors[layer_name]
+ )
+ continue
+ normalized_realized = []
+ for target_name, tensor in sorted(realized.items()):
+ if self._is_weight_converter(mapping):
+ target_name, _ = self._rename_source_key(target_name, inverted_renamings, [])
+ tensor = tensor[0] if isinstance(tensor, list) else tensor
+ if tensor is None or tensor.numel() == 0:
+ logger.debug(
+ "gguf: falling back to original tensors for empty Transformers weight conversion %s",
+ layer_name,
+ )
+ ordered_tensors.extend(
+ (converter_order[layer_name], fallback) for fallback in converter_fallback_tensors[layer_name]
+ )
+ break
+ normalized_realized.append((target_name, tensor))
+ else:
+ lineage = tuple(dict.fromkeys(converter_lineage[layer_name]))
+ for target_name, tensor in normalized_realized:
+ ordered_tensors.append(
+ (
+ converter_order[layer_name],
+ RestoredTensor(
+ target_name,
+ lambda tensor=tensor: tensor,
+ lineage,
+ "converter",
+ ),
+ )
+ )
+
+ for _, restored in sorted(ordered_tensors, key=lambda item: item[0]):
+ yield restored
+
+ def _get_weight_conversions(self):
+ weight_conversions = getattr(self.model, "_weight_conversions", None)
+ if weight_conversions is not None:
+ return weight_conversions
+
+ try:
+ from transformers.conversion_mapping import get_model_conversion_mapping
+ from transformers.core_model_loading import PrefixChange
+ except Exception:
+ return None
+
+ try:
+ conversions = get_model_conversion_mapping(self.model, add_legacy=False)
+ except Exception as exc:
+ logger.warning("gguf: unable to get Transformers weight conversion mapping: %s", exc)
+ return None
+
+ # Match Transformers' own fallback in revert_weight_conversion: if the
+ # model was not loaded through from_pretrained, do not invent prefix adds.
+ conversions = [conversion for conversion in conversions if not isinstance(conversion, PrefixChange)]
+ return conversions or None
+
+ def _get_inverted_transforms(self, weight_conversions):
+ inverted = []
+ for transform in weight_conversions[::-1]:
+ try:
+ reversed_transform = transform.reverse_transform()
+ except Exception as exc:
+ logger.warning("gguf: skipping non-reversible Transformers weight transform %s: %s", transform, exc)
+ continue
+ if self._is_weight_converter(reversed_transform) and not self._is_supported_converter(reversed_transform):
+ logger.warning("gguf: skipping unsupported Transformers weight converter %s", reversed_transform)
+ continue
+ inverted.append(reversed_transform)
+ return inverted
+
+ def _is_supported_converter(self, converter):
+ return all(type(op).__name__ in self._SUPPORTED_OPS for op in getattr(converter, "operations", ()))
+
+ @staticmethod
+ def _is_weight_converter(transform):
+ try:
+ from transformers.core_model_loading import WeightConverter
+ except Exception:
+ return False
+ return isinstance(transform, WeightConverter)
+
+ @staticmethod
+ def _rename_source_key(source_key, weight_renamings, weight_converters):
+ from transformers.core_model_loading import rename_source_key
+
+ return rename_source_key(source_key, weight_renamings, weight_converters)
diff --git a/auto_round/export/export_to_gguf/llama_cpp_conversion.py b/auto_round/export/export_to_gguf/llama_cpp_conversion.py
index 385fc47ec..e057dbbe7 100644
--- a/auto_round/export/export_to_gguf/llama_cpp_conversion.py
+++ b/auto_round/export/export_to_gguf/llama_cpp_conversion.py
@@ -17,6 +17,7 @@ from __future__ import annotations
import ast
import importlib
import json
+import logging
import os
import shutil
import sys
@@ -113,11 +114,21 @@ def _clear_loaded_conversion_modules() -> None:
del sys.modules[name]
+def _configure_conversion_logger() -> None:
+ conversion_logger = logging.getLogger("hf-to-gguf")
+ conversion_logger.setLevel(logger.level)
+ conversion_logger.propagate = False
+ for handler in logger.handlers:
+ if handler not in conversion_logger.handlers:
+ conversion_logger.addHandler(handler)
+
+
def _import_conversion_from(root: Path, source: str) -> ConversionContext:
global _ACTIVE_CONVERSION_ROOT, _CONVERSION_MODULE, _CONVERSION_SOURCE
root = root.resolve()
if _CONVERSION_MODULE is not None and _ACTIVE_CONVERSION_ROOT == root:
+ _configure_conversion_logger()
return ConversionContext(_CONVERSION_MODULE, _CONVERSION_SOURCE)
_clear_loaded_conversion_modules()
@@ -133,6 +144,7 @@ def _import_conversion_from(root: Path, source: str) -> ConversionContext:
_ACTIVE_CONVERSION_ROOT = root
_CONVERSION_MODULE = module
_CONVERSION_SOURCE = source
+ _configure_conversion_logger()
return ConversionContext(module, source)
diff --git a/docs/auto_scheme_acc.md b/docs/auto_scheme_acc.md
index 50d64f90b..ad404c313 100644
--- a/docs/auto_scheme_acc.md
+++ b/docs/auto_scheme_acc.md
@@ -9,7 +9,7 @@ python3 -m auto_round Qwen/Qwen3-8B --options "gguf:q2_k_s,gguf:q4_k_s"
eval
~~~
- llm serve ./test_gguf/Qwen3-8B-w2g16/ --port 8001 --max-model-len 8192 --host 127.0.0.1 --served-model-name qwen
+ vllm serve ./test_gguf/Qwen3-8B-w2g16/ --port 8000 --max-model-len 8192 --host 127.0.0.1 --served-model-name qwen
evalscope eval --model qwen --api-url http://127.0.0.1:8000/v1 --api-key EMPTY --datasets math_500 gpqa_diamoid mmlu_pro --eval-batch-size 32
~~~
diff --git a/test/test_cpu/advanced/test_evaluation_functions.py b/test/test_cpu/advanced/test_evaluation_functions.py
index 9cc99311d..d6476ec21 100644
--- a/test/test_cpu/advanced/test_evaluation_functions.py
+++ b/test/test_cpu/advanced/test_evaluation_functions.py
@@ -25,6 +25,33 @@ from unittest.mock import MagicMock, patch
import pytest
+class TestSelectGgufEvalFile:
+ """Test GGUF text file selection for evaluation."""
+
+ def test_selects_only_text_gguf_when_mixed_format_filename_differs(self, tmp_path):
+ from auto_round.eval.evaluation import select_gguf_eval_file
+
+ (tmp_path / "gemma-4-E4B-it-7.5B-Q2_K_S.gguf").touch()
+ (tmp_path / "mmproj-model.gguf").touch()
+
+ gguf_file, candidates = select_gguf_eval_file(str(tmp_path), ["gguf:q2_k_mixed"])
+
+ assert gguf_file == "gemma-4-E4B-it-7.5B-Q2_K_S.gguf"
+ assert candidates == ["gemma-4-E4B-it-7.5B-Q2_K_S.gguf"]
+
+ def test_prefers_exact_format_match_when_multiple_text_gguf_files_exist(self, tmp_path):
+ from auto_round.eval.evaluation import select_gguf_eval_file
+
+ (tmp_path / "model-Q4_0.gguf").touch()
+ (tmp_path / "model-Q8_0.gguf").touch()
+ (tmp_path / "mmproj-model.gguf").touch()
+
+ gguf_file, candidates = select_gguf_eval_file(str(tmp_path), ["gguf:q4_0"])
+
+ assert gguf_file == "model-Q4_0.gguf"
+ assert candidates == ["model-Q4_0.gguf", "model-Q8_0.gguf"]
+
+
class TestParseVllmArgs:
"""Test parse_vllm_args function for parsing custom vllm arguments."""
diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.py
index 1e5f54937..d9c04171d 100644
--- a/test/test_cpu/export/test_gguf_format.py
+++ b/test/test_cpu/export/test_gguf_format.py
@@ -201,6 +201,143 @@ class TestGGUF:
else:
assert abs(file_size - 264) < 5.0
+ def test_mmproj_uses_native_f32_export_when_nontext_is_not_quantized(self, tmp_path, monkeypatch):
+ from auto_round.export.export_to_gguf import export
+ from auto_round.export.export_to_gguf.config import ModelType
+
+ ftypes = {
+ "q4_0": object(),
+ "f32": object(),
+ }
+ wrapper_calls = []
+
+ class FakeConfig:
+ model_type = "qwen2_vl"
+
+ class FakeModel:
+ name_or_path = str(tmp_path)
+ config = FakeConfig()
+
+ class FakeModelClass:
+ def __init__(self, **kwargs):
+ self.__dict__.update(kwargs)
+ self.model_arch = "MMPROJ"
+
+ class FakeModelBase:
+ @staticmethod
+ def load_hparams(_dir_model, _is_mistral_format):
+ return {"architectures": ["FakeArchitecture"], "quantization_config": {"bits": 4}}
+
+ class FakeConversion:
+ ModelBase = FakeModelBase
+
+ @staticmethod
+ def model_type(model_type):
+ return model_type
+
+ @staticmethod
+ def get_model_architecture(_hparams, _model_type):
+ return "FakeArchitecture"
+
+ @staticmethod
+ def get_model_class(_model_architecture, model_type):
+ return FakeModelClass
+
+ def fake_wrapper(model_instance, **kwargs):
+ wrapper_calls.append(model_instance)
+ model_instance.was_wrapped = True
+ model_instance.model = kwargs["model"]
+ return model_instance
+
+ monkeypatch.setattr(export, "FTYPE_MAP", ftypes)
+ monkeypatch.setattr(export, "get_conversion", lambda *_args, **_kwargs: FakeConversion)
+ monkeypatch.setattr(export, "wrapper_model_instance", fake_wrapper)
+ monkeypatch.setattr(export, "handle_special_model", lambda model_instance, _architecture: model_instance)
+
+ model_instance = export.create_model_class(
+ output_dir=str(tmp_path),
+ model=FakeModel(),
+ layer_config={},
+ backend="gguf:q4_0",
+ model_type=ModelType.MMPROJ,
+ quant_nontext_module=False,
+ )
+
+ assert wrapper_calls == []
+ assert not hasattr(model_instance, "model")
+ assert model_instance.ftype is ftypes["f32"]
+ assert model_instance.fname_out == tmp_path / "mmproj-model.gguf"
+
+ def test_text_still_uses_autoround_wrapper_when_nontext_is_not_quantized(self, tmp_path, monkeypatch):
+ from auto_round.export.export_to_gguf import export
+ from auto_round.export.export_to_gguf.config import ModelType
+
+ ftypes = {
+ "q4_0": object(),
+ "f32": object(),
+ }
+ wrapper_calls = []
+
+ class FakeConfig:
+ model_type = "qwen2_vl"
+
+ class FakeModel:
+ name_or_path = str(tmp_path)
+ config = FakeConfig()
+
+ class FakeModelClass:
+ def __init__(self, **kwargs):
+ self.__dict__.update(kwargs)
+ self.model_arch = "TEXT"
+
+ class FakeModelBase:
+ @staticmethod
+ def load_hparams(_dir_model, _is_mistral_format):
+ return {"architectures": ["FakeArchitecture"], "quantization_config": {"bits": 4}}
+
+ class FakeConversion:
+ ModelBase = FakeModelBase
+
+ @staticmethod
+ def model_type(model_type):
+ return model_type
+
+ @staticmethod
+ def get_model_architecture(_hparams, _model_type):
+ return "FakeArchitecture"
+
+ @staticmethod
+ def get_model_class(_model_architecture, model_type):
+ return FakeModelClass
+
+ def fake_wrapper(model_instance, **kwargs):
+ wrapper_calls.append((model_instance, kwargs))
+ model_instance.was_wrapped = True
+ model_instance.model = kwargs["model"]
+ return model_instance
+
+ monkeypatch.setattr(export, "FTYPE_MAP", ftypes)
+ monkeypatch.setattr(export, "get_conversion", lambda *_args, **_kwargs: FakeConversion)
+ monkeypatch.setattr(export, "wrapper_model_instance", fake_wrapper)
+ monkeypatch.setattr(export, "handle_special_model", lambda model_instance, _architecture: model_instance)
+
+ model_instance = export.create_model_class(
+ output_dir=str(tmp_path),
+ model=FakeModel(),
+ layer_config={},
+ backend="gguf:q4_0",
+ model_type=ModelType.TEXT,
+ quant_nontext_module=False,
+ )
+
+ assert len(wrapper_calls) == 1
+ assert wrapper_calls[0][0] is model_instance
+ assert wrapper_calls[0][1]["quant_nontext_module"] is False
+ assert model_instance.was_wrapped
+ assert isinstance(model_instance.model, FakeModel)
+ assert model_instance.ftype is ftypes["q4_0"]
+ assert model_instance.fname_out == tmp_path
+
def test_qtype_setting(self, tiny_qwen_vl_model_path):
# Qwen2.5-0.5B-Instruct no output, token_embed q6_k fallbakc to q8_0 336M
# Qwen3-0.6B output q6_k, token_embed q4_0 448M
@@ -321,8 +458,8 @@ class TestGGUF:
gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))
assert gguf_model.get_tensor(2).name == "blk.0.attn_k.weight"
assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"
- assert gguf_model.get_tensor(9).name == "blk.0.ffn_up_exps.weight"
- assert gguf_model.get_tensor(9).tensor_type.name == "Q2_K"
+ tensor_types = {tensor.name: tensor.tensor_type.name for tensor in gguf_model.tensors}
+ assert tensor_types["blk.0.ffn_up_exps.weight"] == "Q2_K"
def test_q2k_mixed_keeps_only_three_dim_expert_weights_at_q2k(self, tiny_qwen_moe_model_path):
model_name = tiny_qwen_moe_model_path
diff --git a/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py
new file mode 100644
index 000000000..d06c1f278
--- /dev/null
+++ b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py
@@ -0,0 +1,355 @@
+import pytest
+import torch
+from torch import nn
+from transformers.core_model_loading import Chunk, PrefixChange, WeightConverter
+
+from auto_round.export.export_to_gguf.hf_checkpoint_restorer import HFCheckpointRestorer
+
+
+class _DummyConfig:
+ model_type = "dummy"
+
+
+class _DummyModel:
+ config = _DummyConfig()
+
+ def __init__(self, state_dict, weight_conversions):
+ self._state_dict = state_dict
+ self._weight_conversions = weight_conversions
+
+ def state_dict(self):
+ return self._state_dict
+
+
+def test_restorer_reverses_transformers_prefix_change():
+ model = _DummyModel(
+ {
+ "model.layers.0.mlp.down_proj.weight": torch.ones(2, 2),
+ },
+ [PrefixChange(prefix_to_remove="language_model", model_prefix="model")],
+ )
+
+ restored = list(HFCheckpointRestorer(model).iter_tensors())
+
+ assert [tensor.checkpoint_name for tensor in restored] == [
+ "model.language_model.layers.0.mlp.down_proj.weight",
+ ]
+ assert restored[0].hf_names == ("model.layers.0.mlp.down_proj.weight",)
+ assert torch.equal(restored[0].tensor_fn(), torch.ones(2, 2))
+
+
+def test_restorer_concats_split_hf_tensors_back_to_checkpoint_tensor():
+ model = _DummyModel(
+ {
+ "model.layers.0.mlp.gate_proj.weight": torch.ones(1, 2),
+ "model.layers.0.mlp.up_proj.weight": torch.full((1, 2), 2.0),
+ },
+ [
+ WeightConverter(
+ source_patterns="mlp.gate_up_proj.weight",
+ target_patterns=["mlp.gate_proj.weight", "mlp.up_proj.weight"],
+ operations=[Chunk(dim=0)],
+ ),
+ ],
+ )
+
+ restored = list(HFCheckpointRestorer(model).iter_tensors())
+
+ assert [tensor.checkpoint_name for tensor in restored] == [
+ "model.layers.0.mlp.gate_up_proj.weight",
+ ]
+ assert restored[0].hf_names == (
+ "model.layers.0.mlp.gate_proj.weight",
+ "model.layers.0.mlp.up_proj.weight",
+ )
+ assert torch.equal(restored[0].tensor_fn(), torch.tensor([[1.0, 1.0], [2.0, 2.0]]))
+
+
+def test_restorer_preserves_model_order_across_converted_and_passthrough_tensors():
+ model = _DummyModel(
+ {
+ "model.layers.0.self_attn.gate_proj.weight": torch.ones(1, 2),
+ "model.layers.0.self_attn.up_proj.weight": torch.full((1, 2), 2.0),
+ "model.layers.0.mlp.experts.weight": torch.ones(2, 2),
+ },
+ [
+ WeightConverter(
+ source_patterns="self_attn.gate_up_proj.weight",
+ target_patterns=["self_attn.gate_proj.weight", "self_attn.up_proj.weight"],
+ operations=[Chunk(dim=0)],
+ ),
+ ],
+ )
+
+ restored = list(HFCheckpointRestorer(model).iter_tensors())
+
+ assert [tensor.checkpoint_name for tensor in restored] == [
+ "model.layers.0.self_attn.gate_up_proj.weight",
+ "model.layers.0.mlp.experts.weight",
+ ]
+
+
+def test_restorer_does_not_rerun_completed_conversion_group(monkeypatch):
+ state_dict = {
+ "model.layers.0.mlp.gate_proj.weight": torch.ones(1, 2),
+ "model.layers.0.mlp.up_proj.weight": torch.full((1, 2), 2.0),
+ }
+ model = _DummyModel(
+ state_dict,
+ [
+ WeightConverter(
+ source_patterns="mlp.gate_up_proj.weight",
+ target_patterns=["mlp.gate_proj.weight", "mlp.up_proj.weight"],
+ operations=[Chunk(dim=0)],
+ ),
+ ],
+ )
+
+ def fail_if_converted(*args, **kwargs):
+ raise AssertionError("completed conversion group was executed again")
+
+ monkeypatch.setattr(WeightConverter, "convert", fail_if_converted)
+
+ restored = list(HFCheckpointRestorer(model, completed_hf_names=set(state_dict)).iter_tensors())
+
+ assert restored == []
+
+
+def test_resolve_restored_qtype_uses_consistent_multi_source_layer_config():
+ import gguf
+
+ from auto_round.export.export_to_gguf.convert import resolve_restored_qtype
+
+ diagnostics = []
+ qtype = resolve_restored_qtype(
+ layer_config={
+ "model.layers.0.mlp.gate_proj": {"bits": 4, "super_bits": 6, "sym": False},
+ "model.layers.0.mlp.up_proj": {"bits": 4, "super_bits": 6, "sym": False},
+ },
+ hf_names=(
+ "model.layers.0.mlp.gate_proj.weight",
+ "model.layers.0.mlp.up_proj.weight",
+ ),
+ checkpoint_name="model.layers.0.mlp.gate_up_proj.weight",
+ gguf_name="blk.0.ffn_gate_up.weight",
+ fallback_qtype=gguf.GGMLQuantizationType.Q5_K,
+ diagnostics=diagnostics,
+ )
+
+ assert qtype == gguf.GGMLQuantizationType.Q4_K
+ assert diagnostics == []
+
+
+def test_resolve_restored_qtype_falls_back_and_records_conflicting_multi_source_layer_config():
+ import gguf
+
+ from auto_round.export.export_to_gguf.convert import resolve_restored_qtype
+
+ diagnostics = []
+ qtype = resolve_restored_qtype(
+ layer_config={
+ "model.layers.0.mlp.gate_proj": {"bits": 4, "super_bits": 6, "sym": False},
+ "model.layers.0.mlp.up_proj": {"bits": 6, "super_bits": 8, "sym": True},
+ },
+ hf_names=(
+ "model.layers.0.mlp.gate_proj.weight",
+ "model.layers.0.mlp.up_proj.weight",
+ ),
+ checkpoint_name="model.layers.0.mlp.gate_up_proj.weight",
+ gguf_name="blk.0.ffn_gate_up.weight",
+ fallback_qtype=gguf.GGMLQuantizationType.Q5_K,
+ diagnostics=diagnostics,
+ )
+
+ assert qtype is None
+ assert diagnostics == [
+ {
+ "checkpoint_name": "model.layers.0.mlp.gate_up_proj.weight",
+ "gguf_name": "blk.0.ffn_gate_up.weight",
+ "reason": "multi_source_dtype_conflict",
+ "fallback_qtype": "Q5_K",
+ "hf_sources": [
+ {
+ "name": "model.layers.0.mlp.gate_proj.weight",
+ "layer_config_qtype": "Q4_K",
+ },
+ {
+ "name": "model.layers.0.mlp.up_proj.weight",
+ "layer_config_qtype": "Q6_K",
+ },
+ ],
+ }
+ ]
+
+
+def test_restorer_falls_back_to_original_tensor_when_weight_conversion_fails():
+ model = _DummyModel(
+ {
+ "model.layers.0.mlp.experts.gate_up_proj": torch.tensor([], dtype=torch.bfloat16),
+ },
+ [
+ WeightConverter(
+ source_patterns="mlp.experts.gate_up_proj",
+ target_patterns=["mlp.experts.*.gate_proj.weight", "mlp.experts.*.up_proj.weight"],
+ operations=[Chunk(dim=1)],
+ ),
+ ],
+ )
+
+ restored = list(HFCheckpointRestorer(model).iter_tensors())
+
+ assert [tensor.checkpoint_name for tensor in restored] == ["model.layers.0.mlp.experts.gate_up_proj"]
+ assert restored[0].transform_kind == "passthrough"
+
+
+def test_get_restored_tensors_replaces_empty_live_tensor_from_checkpoint(monkeypatch):
+ from auto_round.export.export_to_gguf import convert
+
+ tensor_name = "model.layers.0.self_attn.q_proj.weight"
+ model = _DummyModel({tensor_name: torch.tensor([], dtype=torch.bfloat16)}, [])
+ model.tensor_name_list = [tensor_name]
+
+ def iter_checkpoint_tensors(cls):
+ if tensor_name not in cls.model.tensor_name_list:
+ yield tensor_name, torch.ones(2, 2)
+
+ monkeypatch.setattr(convert, "_iter_extra_tensors", iter_checkpoint_tensors)
+
+ class Converter:
+ pass
+
+ converter = Converter()
+ converter.model = model
+ restored = list(convert.get_restored_tensors(converter))
+
+ assert [tensor.checkpoint_name for tensor in restored] == [tensor_name]
+ assert restored[0].hf_names == (tensor_name,)
+ assert torch.equal(restored[0].tensor_fn(), torch.ones(2, 2))
+
+
+def test_name_resolution_diagnostics_are_never_written_to_export_directory(tmp_path, monkeypatch):
+ from auto_round.export.export_to_gguf.convert import _flush_name_resolution_diagnostics
+
+ monkeypatch.setenv("AR_GGUF_DEBUG_NAME_RESOLUTION", "1")
+
+ class Converter:
+ fname_out = tmp_path / "model.gguf"
+ _gguf_name_resolution_diagnostics = [{"checkpoint_name": "x"}]
+
+ _flush_name_resolution_diagnostics(Converter)
+
+ assert not (tmp_path / "gguf_name_resolution.json").exists()
+
+
+def test_gguf_writer_has_tensor_supports_current_gguf_dict_entries():
+ import numpy as np
+ from gguf.gguf_writer import GGUFWriter
+
+ from auto_round.export.export_to_gguf.convert import _gguf_writer_has_tensor
+
+ writer = GGUFWriter(path=None, arch="llama")
+ writer.add_tensor("blk.24.attn_k_norm.weight", np.zeros((1,), dtype=np.float32))
+
+ assert _gguf_writer_has_tensor(writer, "blk.24.attn_k_norm.weight")
+ assert not _gguf_writer_has_tensor(writer, "blk.24.attn_v_norm.weight")
+
+
+def test_gguf_shape_fallback_updates_layer_config_before_quantization():
+ from auto_round.compressors.utils import _apply_gguf_shape_fallback
+ from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG
+
+ class Model(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = nn.Linear(1152, 4096, bias=False)
+
+ layer_config = {"proj": {**GGUF_INNER_CONFIG["gguf:q6_k"], "fixed_by_user": True, "scale_dtype": torch.float32}}
+
+ _apply_gguf_shape_fallback(layer_config, Model())
+
+ assert layer_config["proj"]["bits"] == 8
+ assert layer_config["proj"]["group_size"] == 32
+ assert layer_config["proj"]["super_bits"] is None
+ assert layer_config["proj"]["super_group_size"] is None
+ assert layer_config["proj"]["fixed_by_user"] is True
+ assert layer_config["proj"]["scale_dtype"] is torch.float32
+
+
+def test_quant_data_recomputes_scale_when_final_qtype_differs_from_layer_config(monkeypatch):
+ import gguf
+ import numpy as np
+
+ from auto_round.export.export_to_gguf import convert
+
+ captured = {}
+
+ class Module:
+ def __init__(self):
+ self.weight = torch.ones(2, 32)
+ self.scale = torch.ones(2, 16)
+
+ module = Module()
+
+ class Converter:
+ model = object()
+ layer_config = {"model.layers.0.self_attn.q_proj": {"bits": 6, "super_bits": 8, "sym": True}}
+
+ def fake_get_module(_model, _layer_name):
+ return module
+
+ def fake_ggml_quant(data_torch, ggml_type, device=None, **kwargs):
+ captured["data_shape"] = tuple(data_torch.shape)
+ captured["ggml_type"] = ggml_type
+ captured["scale"] = kwargs["scale"]
+ return np.zeros((data_torch.shape[0], data_torch.shape[1]), dtype=np.float32)
+
+ monkeypatch.setattr(convert, "get_module", fake_get_module)
+ monkeypatch.setattr(convert, "ggml_quant", fake_ggml_quant)
+
+ data, qtype = convert._quant_data(
+ Converter(),
+ torch.ones(2, 32),
+ gguf.GGMLQuantizationType.Q8_0,
+ "model.layers.0.self_attn.q_proj.weight",
+ "model.layers.0.self_attn.q_proj.weight",
+ "blk.0.attn_q.weight",
+ 0,
+ device="cpu",
+ )
+
+ assert qtype == gguf.GGMLQuantizationType.Q8_0
+ assert data.shape == (2, 32)
+ assert captured["ggml_type"] == "q8_0"
+ assert captured["data_shape"] == (2, 32)
+ assert captured["scale"] is None
+
+
+def test_quant_data_rejects_layer_scale_that_does_not_match_final_qtype(monkeypatch):
+ import gguf
+
+ from auto_round.export.export_to_gguf import convert
+
+ class Module:
+ def __init__(self):
+ self.weight = torch.ones(2, 32)
+ self.scale = torch.ones(2, 2)
+
+ module = Module()
+
+ class Converter:
+ model = object()
+ layer_config = {"model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 32, "sym": True}}
+
+ monkeypatch.setattr(convert, "get_module", lambda _model, _layer_name: module)
+
+ with pytest.raises(ValueError, match="q8_0 scale shape"):
+ convert._quant_data(
+ Converter(),
+ torch.ones(2, 32),
+ gguf.GGMLQuantizationType.Q8_0,
+ "model.layers.0.self_attn.q_proj.weight",
+ "model.layers.0.self_attn.q_proj.weight",
+ "blk.0.attn_q.weight",
+ 0,
+ device="cpu",
+ )
diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.py
index 458266b86..2c76f0671 100644
--- a/test/test_cuda/export/test_gguf_format.py
+++ b/test/test_cuda/export/test_gguf_format.py
@@ -175,7 +175,7 @@ class TestAutoRound:
print(f"{file}: {os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2} MB")
file_size = os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2
if "mmproj-model.gguf" in file:
- assert abs(file_size - 75) < 5.0
+ assert abs(file_size - 62) < 5.0
else:
assert abs(file_size - 690) < 10.0