PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix few gguf bugs of hf-name and checkpoint name (#1939)
Heng Guo126793d50cf8
diff --git a/auto_round/compressors/entry.py b/auto_round/compressors/entry.pyindex 23d11518e..1770b88b4 100644--- a/auto_round/compressors/entry.py+++ b/auto_round/compressors/entry.py@@ -362,7 +362,8 @@ class AutoRound(object):# Model-free routing is now supported directly by the new entry path.model_free_iters = 0 if isinstance(quant_config, RTNConfig) else getattr(quant_config, "iters", None)model_free_disable_opt_rtn = getattr(quant_config, "disable_opt_rtn", None)- if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_kwargs):+ route_decision_kwargs = dict(route_kwargs, format=format)+ if is_model_free_route(model, scheme, model_free_iters, model_free_disable_opt_rtn, route_decision_kwargs):from auto_round.compressors.model_free import ModelFreeCompressorif not isinstance(model, str):diff --git a/auto_round/compressors/model_free.py b/auto_round/compressors/model_free.pyindex b94a81fab..5d2f33453 100644--- a/auto_round/compressors/model_free.py+++ b/auto_round/compressors/model_free.py@@ -1948,6 +1948,11 @@ class ModelFreeCompressor(_ModelFreeCompressorCore):# Start from the remaining user kwargs and explicitly set/override# known compressor init parameters for clarity.fallback_init = dict(fallback_kwargs)+ # Route-control kwargs are only meaningful for the initial entry+ # selection. Strip them so fallback always re-enters the regular flow+ # with a single explicit disable_model_free=True override.+ fallback_init.pop("model_free", None)+ fallback_init.pop("disable_model_free", None)fallback_init.update(model=model_name_or_path,iters=0,diff --git a/auto_round/compressors/utils.py b/auto_round/compressors/utils.pyindex a002fbfb2..74d851aca 100644--- a/auto_round/compressors/utils.py+++ b/auto_round/compressors/utils.py@@ -611,6 +611,7 @@ def set_layer_config(model_type = ModelType.MMPROJ if is_mllm else ModelType.TEXTlayer_config, _ = get_layer_config_by_gguf_format(layer_config, gguf_name.lower(), model, model_type)+ _apply_gguf_shape_fallback(layer_config, model)dispatch_layer_config(layer_config)return layer_config, has_qlayer_outside_block, regex_config@@ -681,6 +682,48 @@ def get_gguf_qtype_by_layer_config(layer_config):raise ValueError("Unknown layer config")+def _apply_gguf_shape_fallback(layer_config, model):+ from auto_round.utils.model import get_module++ for layer_name, config in layer_config.items():+ if not check_to_quantized(config):+ continue+ layer = get_module(model, layer_name)+ if layer is None or not hasattr(layer, "weight"):+ continue+ try:+ qtype = get_gguf_qtype_by_layer_config(config)+ except ValueError:+ continue+ if qtype is None:+ continue++ gguf_type = f"gguf:{qtype.name.lower()}"+ block_size = GGML_QUANT_SIZES[gguf_type.split(":")[-1]][0]+ input_features = (+ layer.weight.shape[0] if type(layer) == transformers.pytorch_utils.Conv1D else layer.weight.shape[-1]+ )+ if input_features % block_size == 0:+ continue++ fallback_type = _gguf_type_fallback(gguf_type)+ fallback_block_size = GGML_QUANT_SIZES[fallback_type.split(":")[-1]][0]+ if input_features % fallback_block_size != 0:+ fallback_type = "gguf:bf16"++ preserved = {key: config[key] for key in ("fixed_by_user", "scale_dtype") if key in config}+ config.update(GGUF_INNER_CONFIG[fallback_type])+ config.update(preserved)+ logger.warning(+ "fallback %s to %s before quantization, because input_features(%s) is not divisible by %s block_size(%s)",+ layer_name,+ fallback_type,+ input_features,+ gguf_type,+ block_size,+ )++def _get_digital_in_layer_name(layer_name):pattern = re.compile(r"([a-zA-Z]+\.){1,}(\d+)")res = re.search(pattern, layer_name)diff --git a/auto_round/eval/evaluation.py b/auto_round/eval/evaluation.pyindex be9baadfb..f6a4bd2a9 100644--- a/auto_round/eval/evaluation.py+++ b/auto_round/eval/evaluation.py@@ -184,6 +184,28 @@ def evaluate_diffusion_model(args, autoround=None, model=None, pipe=None):diffusion_eval(pipe, args.prompt_file, metrics, args.image_save_dir, 1, gen_kwargs, args.limit)+def select_gguf_eval_file(eval_folder, formats):+ """Select the text GGUF file used for HF-based evaluation."""+ gguf_format = None+ for format in formats:+ if format.startswith("gguf"):+ gguf_format = format.split(":")[-1].upper()+ break++ if gguf_format is None:+ return None, []++ gguf_files = sorted(+ file for file in os.listdir(eval_folder) if file.endswith(".gguf") and file != "mmproj-model.gguf"+ )+ matched_files = [file for file in gguf_files if gguf_format in file]+ if matched_files:+ return matched_files[0], gguf_files+ if len(gguf_files) == 1:+ return gguf_files[0], gguf_files+ return None, gguf_files++def load_gguf_model_for_eval(eval_folder, formats, args):"""Load GGUF model for evaluation.@@ -202,26 +224,13 @@ def load_gguf_model_for_eval(eval_folder, formats, args):from auto_round.utils import get_model_dtype, logger- # Find corresponding GGUF format- gguf_format = None- for format in formats:- if format.startswith("gguf"):- gguf_format = format.split(":")[-1].upper()- break-- if gguf_format is None:+ gguf_file, gguf_files = select_gguf_eval_file(eval_folder, formats)+ if not any(format.startswith("gguf") for format in formats):logger.error("No valid gguf format found in formats. Please check the input.")sys.exit(-1)- # Find matching GGUF file- gguf_file = None- for file in os.listdir(eval_folder):- if gguf_format in file:- gguf_file = file- break-if gguf_file is None:- logger.error("Cannot find correct gguf file for evaluation, please check.")+ logger.error("Cannot find correct gguf file for evaluation, candidates=%s", gguf_files)sys.exit(-1)# Load model and tokenizer@@ -482,6 +491,8 @@ def run_model_evaluation(model, tokenizer, autoround, folders, formats, args):# Load or prepare model instanceif eval_gguf_model:model, tokenizer = load_gguf_model_for_eval(eval_folder, formats, args)+ if model is None:+ returnelse:eval_model_dtype = get_model_dtype(args.eval_model_dtype, "auto")model = prepare_model_for_eval(model, args.device_map, eval_model_dtype)diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 61ac14bb5..be21e33e4 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -39,7 +39,6 @@ import osimport refrom functools import partialfrom itertools import chain-from pathlib import Pathfrom typing import TYPE_CHECKING, Any, Iteratorimport numpy as np@@ -48,6 +47,7 @@ from transformers import AutoConfigfrom auto_round.export.export_to_gguf.config import ModelTypefrom auto_round.export.export_to_gguf.gguf_dtype import GGUFDTypeSelector+from auto_round.export.export_to_gguf.hf_checkpoint_restorer import HFCheckpointRestorer, RestoredTensorfrom auto_round.export.export_to_gguf.packing import ggml_quantfrom auto_round.utils import (LazyImport,@@ -75,8 +75,10 @@ def wrapper_model_instance(model_instance.model = modelmodel_instance.layer_config = layer_configmodel_instance.low_cpu_mem_usage = _need_low_cpu_mem(low_cpu_mem_usage)+ model_instance._gguf_written_hf_names = set()+ model_instance._gguf_written_checkpoint_names = set()- model_instance.get_tensors = partial(get_tensors, model_instance)+ model_instance.get_tensors = partial(get_restored_tensors, model_instance)model_instance.prepare_tensors = partial(prepare_tensors, model_instance)model_instance.device = device@@ -130,30 +132,11 @@ def is_mmproj_tensor_name(name):return any(key in name for key in MM_KEYS)-def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:- if not hasattr(cls.model, "tensor_name_list"):- cls.model.tensor_name_list = []-- checkpoint_conversion_mapping = getattr(cls.model, "_checkpoint_conversion_mapping", {}) or {}- reverse_key_mapping = {v: k for k, v in checkpoint_conversion_mapping.items()}- for name, tensor in cls.model.named_parameters():- if name not in cls.model.tensor_name_list:- cls.model.tensor_name_list.append(name)- for pattern, replacement in reverse_key_mapping.items():- replacement = replacement.lstrip("^") # strip off un-needed chars and patterns- replacement = re.sub(r"\(.*\)", "", replacement)- key, n_replace = re.subn(pattern, replacement, name)- cls.model.tensor_name_list.append(key)- if n_replace > 0:- break- yield name, tensor-+def _iter_extra_tensors(cls):def is_extra_tensor(tensor_name):if getattr(cls.model, "_is_quantized_input_module", False) and "scale" in tensor_name.split(".")[-1]:return False- if tensor_name not in cls.model.tensor_name_list:- return True- return False+ return tensor_name not in getattr(cls.model, "tensor_name_list", [])extra_tensor = {}if hasattr(cls.model, "name_or_path"):@@ -173,13 +156,43 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:if is_extra_tensor(tensor_name):extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)else:- f = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")- for tensor_name in f.keys():- if is_extra_tensor(tensor_name):- extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)-- for name, tensor in extra_tensor.items():- yield name, tensor+ model_file = os.path.join(dir_path, "model.safetensors")+ if os.path.exists(model_file):+ f = safe_open(model_file, framework="pt")+ for tensor_name in f.keys():+ if is_extra_tensor(tensor_name):+ extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)++ yield from extra_tensor.items()+++def get_restored_tensors(cls) -> Iterator[RestoredTensor]:+ written_hf_names = getattr(cls, "_gguf_written_hf_names", set())+ written_checkpoint_names = getattr(cls, "_gguf_written_checkpoint_names", set())+ cls.model.tensor_name_list = list(written_hf_names | written_checkpoint_names)++ pending_checkpoint_tensors = {}+ for restored in HFCheckpointRestorer(cls.model, completed_hf_names=written_hf_names).iter_tensors():+ tensor = restored.tensor_fn()+ if tensor is None or tensor.numel() == 0:+ pending_checkpoint_tensors[restored.checkpoint_name] = restored+ continue+ for tensor_name in (restored.checkpoint_name, *restored.hf_names):+ if tensor_name not in cls.model.tensor_name_list:+ cls.model.tensor_name_list.append(tensor_name)+ yield RestoredTensor(+ restored.checkpoint_name,+ lambda tensor=tensor: tensor,+ restored.hf_names,+ restored.transform_kind,+ )++ for name, tensor in _iter_extra_tensors(cls):+ pending = pending_checkpoint_tensors.pop(name, None)+ if pending is None:+ yield RestoredTensor(name, lambda tensor=tensor: tensor, (name,), "extra")+ else:+ yield RestoredTensor(name, lambda tensor=tensor: tensor, pending.hf_names, pending.transform_kind)def _quant_data_with_args(@@ -208,6 +221,58 @@ def _quant_data_with_args(return data+def _remap_quant_attr_tensor(cls, attr_tensor, module, modify_name, new_name, bid):+ if not isinstance(attr_tensor, torch.Tensor):+ return attr_tensor+ if not (hasattr(cls, "permute") or need_modify_tensor(cls, modify_name)):+ return attr_tensor++ bs = module.weight.shape[0]+ remapped = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)).get(new_name)+ return attr_tensor if remapped is None else remapped+++def _expected_scale_columns(ggml_type):+ from auto_round.export.export_to_gguf.config import QK_K++ if ggml_type in {"q4_0", "q4_1", "q5_0", "q5_1", "q8_0"}:+ return 1+ if ggml_type in {"q2_k", "q3_k", "q6_k"}:+ return QK_K // 16+ if ggml_type in {"q4_k", "q5_k"}:+ return QK_K // 32+ return None+++def _validate_quant_attr_shapes(data_torch, data_qtype, kwargs, name, new_name):+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES++ ggml_type = data_qtype.name.lower()+ if ggml_type not in GGML_QUANT_SIZES or kwargs.get("scale") is None:+ return++ block_size, _ = GGML_QUANT_SIZES[ggml_type]+ n_blocks = data_torch.numel() // block_size+ scale_cols = _expected_scale_columns(ggml_type)+ if scale_cols is None:+ return++ scale = kwargs["scale"]+ if scale.numel() != n_blocks * scale_cols:+ raise ValueError(+ f"{name} -> {new_name}: {ggml_type} scale shape {tuple(scale.shape)} does not match "+ f"tensor shape {tuple(data_torch.shape)}; expected {n_blocks * scale_cols} scale values"+ )++ for attr in ["d_scale", "d_wmin"]:+ tensor = kwargs.get(attr)+ if tensor is not None and tensor.numel() != n_blocks:+ raise ValueError(+ f"{name} -> {new_name}: {ggml_type} {attr} shape {tuple(tensor.shape)} does not match "+ f"tensor shape {tuple(data_torch.shape)}; expected {n_blocks} values"+ )++def need_modify_tensor(cls, name):hf_arch = getattr(cls, "hf_arch", "")if hf_arch in "Qwen3NextForCausalLM" and "in_proj_qkvz.weight" in name:@@ -215,7 +280,7 @@ def need_modify_tensor(cls, name):return False-def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=None):+def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=None, use_layer_attrs=True):"""Args:@@ -236,14 +301,18 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, delse:layer_name = namemodule = get_module(cls.model, layer_name)- kwargs = {- "scale": module.scale if hasattr(module, "scale") else None,- "zp": module.zp if hasattr(module, "zp") else None,- "d_scale": module.w_d_scale if hasattr(module, "w_d_scale") else None,- "d_wmin": module.w_d_wmin if hasattr(module, "w_d_wmin") else None,- "wmin": module.w_wmin if hasattr(module, "w_wmin") else None,- "imatrix": module.imatrix if hasattr(module, "imatrix") else None,- }+ kwargs = {"scale": None, "zp": None, "d_scale": None, "d_wmin": None, "wmin": None, "imatrix": None}+ source_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype, explicit_only=True)+ use_layer_attrs = use_layer_attrs and source_qtype == data_qtype+ if use_layer_attrs:+ kwargs = {+ "scale": module.scale if hasattr(module, "scale") else None,+ "zp": module.zp if hasattr(module, "zp") else None,+ "d_scale": module.w_d_scale if hasattr(module, "w_d_scale") else None,+ "d_wmin": module.w_d_wmin if hasattr(module, "w_d_wmin") else None,+ "wmin": module.w_wmin if hasattr(module, "w_wmin") else None,+ "imatrix": module.imatrix if hasattr(module, "imatrix") else None,+ }# patch for Qwen3_5, Qwen3_5 handles some weights specially,# but the scale doesn't match; these weights are handled by gguf itself.# Define model architectures that need special handling@@ -268,7 +337,7 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, dhf_arch = getattr(cls, "hf_arch", "")should_skip = hf_arch in QWEN3_5_MODELS and any(key in name for key in QWEN3_5_SKIP_KEYS)- if not should_skip:+ if use_layer_attrs and not should_skip:# support for MOE model with cls experts not linearfor attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:if not (hasattr(module, attr) and getattr(module, attr) is not None):@@ -278,15 +347,13 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, dif not isinstance(attr_tensor, torch.Tensor):continue- if hasattr(cls, "permute") or need_modify_tensor(cls, name):- bs = module.weight.shape[0]- attr_tensors_dict = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid))- attr_tensor = attr_tensors_dict[new_name]+ attr_tensor = _remap_quant_attr_tensor(cls, attr_tensor, module, modify_name, new_name, bid)# Map attribute names to kwargs keys: w_d_scale -> d_scale, w_d_wmin -> d_wmin, w_wmin -> wminkwargs_key = attr.replace("w_", "") if attr.startswith("w_") else attrkwargs[kwargs_key] = attr_tensor.to(torch.float32)data_torch = data_torch.to(torch.float32)+ _validate_quant_attr_shapes(data_torch, data_qtype, kwargs, name, new_name)data = ggml_quant(data_torch, data_qtype.name.lower(), device=device, **kwargs)# else:# # if data_torch.dtype ==torch.float32:@@ -366,6 +433,57 @@ def get_qtype_by_layer_config(layer_config, name, data_qtype, *, explicit_only=F# return data_qtype+def _qtype_name(qtype):+ return qtype.name if hasattr(qtype, "name") else str(qtype)+++def resolve_restored_qtype(+ layer_config,+ hf_names,+ checkpoint_name,+ gguf_name,+ fallback_qtype,+ diagnostics,+):+ source_qtypes = [+ get_qtype_by_layer_config(layer_config, hf_name, fallback_qtype, explicit_only=True) for hf_name in hf_names+ ]+ matched_qtypes = [qtype for qtype in source_qtypes if qtype is not None]++ if len(matched_qtypes) == 1 and len(hf_names) == 1:+ return matched_qtypes[0]+ if len(matched_qtypes) > 0 and len(matched_qtypes) == len(source_qtypes):+ unique_qtypes = set(matched_qtypes)+ if len(unique_qtypes) == 1:+ return matched_qtypes[0]++ if len(hf_names) <= 1 and len(matched_qtypes) == 0:+ reason = "layer_config_miss"+ elif len(matched_qtypes) == 0:+ reason = "multi_source_layer_config_miss"+ elif len(matched_qtypes) != len(source_qtypes):+ reason = "multi_source_partial_layer_config_miss"+ else:+ reason = "multi_source_dtype_conflict"++ diagnostics.append(+ {+ "checkpoint_name": checkpoint_name,+ "gguf_name": gguf_name,+ "reason": reason,+ "fallback_qtype": _qtype_name(fallback_qtype),+ "hf_sources": [+ {+ "name": hf_name,+ "layer_config_qtype": None if qtype is None else _qtype_name(qtype),+ }+ for hf_name, qtype in zip(hf_names, source_qtypes)+ ],+ }+ )+ return None++def _special_name_handle(cls, name):# using transformers >= 5.4, model.language_model.embed_tokens.weight changed to# model.language_model.model.embed_tokens.weight after saved@@ -416,15 +534,45 @@ def _special_name_handle(cls, name):return name+def _to_restored_tensor(item):+ if isinstance(item, RestoredTensor):+ return item+ name, data_torch = item+ return RestoredTensor(name, lambda data_torch=data_torch: data_torch, (name,), "passthrough")+++def filter_restored_tensor(cls, restored):+ filtered = cls.filter_tensors((restored.checkpoint_name, restored.tensor_fn))+ if filtered is None:+ return None+ return RestoredTensor(filtered[0], filtered[1], restored.hf_names, restored.transform_kind)+++def _gguf_writer_has_tensor(gguf_writer, tensor_name):+ for tensor_info in gguf_writer.tensors:+ if isinstance(tensor_info, dict):+ if tensor_name in tensor_info:+ return True+ continue+ existing_name = getattr(tensor_info, "name", None)+ if existing_name is None and isinstance(tensor_info, (list, tuple)) and len(tensor_info) > 0:+ existing_name = tensor_info[0]+ if tensor_name == existing_name:+ return True+ return False++def _count_attention_wv_tensors(cls):count = 0- for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):- if data_torch is None or data_torch.numel() == 0:- continue- filtered = cls.filter_tensors((name, lambda data_torch=data_torch: data_torch))+ for item in chain(cls.generate_extra_tensors(), cls.get_tensors()):+ restored = _to_restored_tensor(item)+ filtered = filter_restored_tensor(cls, restored)if filtered is None:continue- modify_name = _special_name_handle(cls, filtered[0])+ data_torch = filtered.tensor_fn()+ if data_torch is None or data_torch.numel() == 0:+ continue+ modify_name = _special_name_handle(cls, filtered.checkpoint_name)try:new_name = cls.map_tensor_name(modify_name)except Exception:@@ -433,11 +581,20 @@ def _count_attention_wv_tensors(cls):return count or None+def _flush_name_resolution_diagnostics(cls):+ diagnostics = getattr(cls, "_gguf_name_resolution_diagnostics", None)+ if diagnostics is None or len(diagnostics) == 0:+ return+ logger.debug("gguf: recorded %d in-memory name resolution fallback diagnostics", len(diagnostics))++def prepare_tensors(cls):device = get_packing_device(cls.device)if not hasattr(cls, "_gguf_dtype_selector"):cls._gguf_dtype_selector = GGUFDTypeSelector(cls.hparams, cls.ftype, cls.model_arch)cls._gguf_dtype_selector.n_attention_wv = _count_attention_wv_tensors(cls)+ if not hasattr(cls, "_gguf_name_resolution_diagnostics"):+ cls._gguf_name_resolution_diagnostics = []# Handle empty tensor_map for models with block_count=0 (like MobileNetV5)if cls.tensor_map.mapping:@@ -445,12 +602,18 @@ def prepare_tensors(cls):else:max_name_len = len("vision_encoder.weight,") # Default reasonable length- for name, data_torch in chain(cls.generate_extra_tensors(), cls.get_tensors()):+ for item in chain(cls.generate_extra_tensors(), cls.get_tensors()):+ restored = _to_restored_tensor(item)+ name = restored.checkpoint_nameis_mmproj_model = cls.model_arch == gguf.MODEL_ARCH.MMPROJif is_mmproj_model != is_mmproj_tensor_name(name):continue- if name in getattr(cls.model, "_tied_weights_keys", []) and not is_separate_tensor(cls.model, name):+ if any(+ hf_name in getattr(cls.model, "_tied_weights_keys", []) and not is_separate_tensor(cls.model, hf_name)+ for hf_name in restored.hf_names+ ):continue+ data_torch = restored.tensor_fn()if data_torch is None or data_torch.numel() == 0:continue# we don't need these@@ -465,10 +628,12 @@ def prepare_tensors(cls):):continue- filtered = cls.filter_tensors((name, lambda data_torch=data_torch: data_torch))+ filtered = filter_restored_tensor(cls, restored)if filtered is None:continue- name = filtered[0]+ checkpoint_name = filtered.checkpoint_name+ hf_names = filtered.hf_names or (checkpoint_name,)+ hf_name = hf_names[0]old_dtype = data_torch.dtype@@ -485,20 +650,19 @@ def prepare_tensors(cls):clean_weight_list = []- modify_name = _special_name_handle(cls, name)+ modify_name = _special_name_handle(cls, checkpoint_name)+ restored_outputs_completed = Falsefor new_name, data_torch in cls.modify_tensors(data_torch, modify_name, bid):- skip = False- for tensor_info in cls.gguf_writer.tensors:- if new_name in tensor_info:- logger.info(f"{new_name} already add to gguf_writer, skip")- skip = True- break- if skip:+ restored_outputs_completed = True+ if _gguf_writer_has_tensor(cls.gguf_writer, new_name):+ logger.debug("%s already added to gguf_writer, skip", new_name)continue# squeeze is necessary for reloading in transformers.data = data_torch.squeeze()n_dims = len(data.shape)- data_qtype: gguf.GGMLQuantizationType | bool = cls.tensor_force_quant(name, new_name, bid, n_dims)+ data_qtype: gguf.GGMLQuantizationType | bool = cls.tensor_force_quant(+ checkpoint_name, new_name, bid, n_dims+ )# Most of the codebase that takes in 1D tensors or norms only handles F32 tensorsif n_dims <= 1 or new_name.endswith("_norm.weight"):@@ -557,16 +721,36 @@ def prepare_tensors(cls):data_qtype = gguf.GGMLQuantizationType.F16# data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type- # get name by new_name (for experts),- name = get_moe_name(cls, name, new_name)- clean_weight_list.append(name)- # get data_qtype by layer_config- layer_config_qtype = get_qtype_by_layer_config(cls.layer_config, name, data_qtype, explicit_only=True)- # # No override (data_qtype is False), or wants to be quantized (data_qtype is True)- if layer_config_qtype is not None:- data_qtype = layer_config_qtype- elif isinstance(data_qtype, bool):- data_qtype = cls._gguf_dtype_selector.select_qtype(new_name, n_dims)+ if cls.model_arch == gguf.MODEL_ARCH.MMPROJ and cls.quant_nontext_module is False:+ data_qtype = gguf.GGMLQuantizationType.F32+ layer_config_names = hf_names+ else:+ # get name by new_name (for experts),+ layer_config_names = tuple(get_moe_name(cls, source_name, new_name) for source_name in hf_names)+ # get data_qtype by layer_config+ fallback_qtype = (+ cls._gguf_dtype_selector.select_qtype(new_name, n_dims)+ if isinstance(data_qtype, bool)+ else data_qtype+ )+ layer_config_qtype = resolve_restored_qtype(+ cls.layer_config,+ layer_config_names,+ checkpoint_name,+ new_name,+ fallback_qtype,+ (+ cls._gguf_name_resolution_diagnostics+ if restored.transform_kind != "extra" and new_name.endswith(".weight") and n_dims > 1+ else []+ ),+ )+ # # No override (data_qtype is False), or wants to be quantized (data_qtype is True)+ if layer_config_qtype is not None:+ data_qtype = layer_config_qtype+ elif isinstance(data_qtype, bool):+ data_qtype = fallback_qtype+ clean_weight_list.extend(layer_config_names)if isinstance(data_qtype, bool):if cls.ftype == gguf.LlamaFileType.ALL_F32:@@ -614,9 +798,6 @@ def prepare_tensors(cls):elif data_qtype == gguf.GGMLQuantizationType.Q6_K:data_qtype = gguf.GGMLQuantizationType.Q8_0- if cls.model_arch == gguf.MODEL_ARCH.MMPROJ and cls.quant_nontext_module is False:- data_qtype = gguf.GGMLQuantizationType.F32-from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZESif data_qtype.name.lower() in GGML_QUANT_SIZES:@@ -628,6 +809,16 @@ def prepare_tensors(cls):)data_qtype = gguf.GGMLQuantizationType.F16+ if n_dims > 1 and data_torch.numel() >= 100_000_000:+ pre_quant_shape = f"{{{', '.join(str(n) for n in reversed(data_torch.shape))}}}"+ logger.info(+ "Start quantizing large tensor %s: %s --> %s, shape = %s",+ new_name,+ old_dtype,+ data_qtype.name,+ pre_quant_shape,+ )+if isinstance(data_qtype, bool) or data_qtype in [gguf.GGMLQuantizationType.F16,gguf.GGMLQuantizationType.BF16,@@ -647,8 +838,8 @@ def prepare_tensors(cls):data = gguf.quants.quantize(data, data_qtype)else:# for deepseek v2- if name.endswith("kv_b_proj.weight") and cls.model_arch.name in ("DEEPSEEK2", "GLM_DSA"):- layer_name = name[: -len(".weight")]+ if hf_name.endswith("kv_b_proj.weight") and cls.model_arch.name in ("DEEPSEEK2", "GLM_DSA"):+ layer_name = hf_name[: -len(".weight")]module = get_module(cls.model, layer_name)n_head_kv = cls.hparams["num_key_value_heads"]v_head_dim = cls.hparams["v_head_dim"]@@ -676,10 +867,10 @@ def prepare_tensors(cls):data = _quant_data_with_args(data_torch, data_qtype, device=device, **attr_list)# for MOE model- elif len(data_torch.shape) == 3 and len(re.findall(r"\d+", name)) == 2:+ elif len(data_torch.shape) == 3 and len(re.findall(r"\d+", hf_name)) == 2:new_data = []for idx, arr in enumerate(data_torch):- arr_name = name.split(".")+ arr_name = hf_name.split(".")for i in range(len(arr_name) - 1, -1, -1):if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):arr_name[i] = str(idx)@@ -692,7 +883,15 @@ def prepare_tensors(cls):del new_dataelse:data, data_qtype = _quant_data(- cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=device+ cls,+ data_torch,+ data_qtype,+ hf_name,+ modify_name,+ new_name,+ bid,+ device=device,+ use_layer_attrs=len(hf_names) == 1,)shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shape@@ -709,6 +908,10 @@ def prepare_tensors(cls):cls.gguf_writer.add_tensor(new_name, data, raw_dtype=data_qtype)+ if restored_outputs_completed:+ cls._gguf_written_checkpoint_names.add(restored.checkpoint_name)+ cls._gguf_written_hf_names.update(restored.hf_names)+# # save cpu memory, but slowif cls.low_cpu_mem_usage:for weight_name in clean_weight_list:@@ -723,3 +926,5 @@ def prepare_tensors(cls):if cls.model_arch == gguf.MODEL_ARCH.LLAMA and "embed_tokens.weight" in weight_name:continueclean_module_parameter(module, weight_name.split(".")[-1])+ if not (hasattr(cls, "current_packing_block") and cls.current_packing_block is not None):+ _flush_name_resolution_diagnostics(cls)diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex ecc776786..bf9d36c0a 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -64,6 +64,16 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {}+def _use_native_nontext_gguf_export(model_type, quant_nontext_module: bool) -> bool:+ return model_type == ModelType.MMPROJ and not quant_nontext_module+++def _set_mmproj_output_path(model_instance):+ if model_instance.fname_out.is_dir():+ model_instance.fname_out = model_instance.fname_out / "mmproj-model.gguf"+ return model_instance++def create_model_class(output_dir,model,@@ -94,7 +104,8 @@ def create_model_class(else:model_name = model_name[-1]- output_type = backend.split(":")[-1]+ native_nontext_export = _use_native_nontext_gguf_export(model_type, quant_nontext_module)+ output_type = "f32" if native_nontext_export else backend.split(":")[-1]if output_type.lower() not in FTYPE_MAP:raise TypeError(f"{output_type} type is not supported")output_type = FTYPE_MAP.get(output_type.lower())@@ -112,14 +123,18 @@ def create_model_class(dry_run=False,small_first_shard=False,)- model_instance = wrapper_model_instance(- model_instance,- model=model,- layer_config=layer_config,- low_cpu_mem_usage=low_cpu_mem_usage,- device=device,- quant_nontext_module=quant_nontext_module,- )+ if native_nontext_export:+ logger.info("Using native llama.cpp F32 export for non-text GGUF model")+ model_instance = _set_mmproj_output_path(model_instance)+ else:+ model_instance = wrapper_model_instance(+ model_instance,+ model=model,+ layer_config=layer_config,+ low_cpu_mem_usage=low_cpu_mem_usage,+ device=device,+ quant_nontext_module=quant_nontext_module,+ )model_instance = handle_special_model(model_instance, model_architecture)return model_instance@@ -253,6 +268,8 @@ def save_quantized_as_gguf()for gguf_model in gguf_model_instance_global:+ model_kind = "mmproj" if gguf_model.model_arch == gguf.MODEL_ARCH.MMPROJ else "text"+ logger.info("Start writing %s GGUF model to %s", model_kind, gguf_model.fname_out)gguf_model.write()rt = time.time() - stlogger.info(f"Model successfully exported to {gguf_model.fname_out}, running time={rt}")diff --git a/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py b/auto_round/export/export_to_gguf/hf_checkpoint_restorer.pynew file mode 100644index 000000000..8f6b8f14e--- /dev/null+++ b/auto_round/export/export_to_gguf/hf_checkpoint_restorer.py@@ -0,0 +1,203 @@+# Copyright (c) 2024 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++from __future__ import annotations++from collections import defaultdict+from copy import deepcopy+from dataclasses import dataclass+from typing import Callable, Iterator++import torch++from auto_round.utils import logger+++@dataclass(frozen=True)+class RestoredTensor:+ checkpoint_name: str+ tensor_fn: Callable[[], torch.Tensor]+ hf_names: tuple[str, ...]+ transform_kind: str+++class HFCheckpointRestorer:+ """Expose a Transformers model state dict as checkpoint-style tensors.++ Transformers may rename or structurally convert checkpoint tensors while loading+ a model. GGUF conversion expects the original checkpoint namespace, while+ AutoRound layer config is keyed by live HF module names. This restorer keeps+ both names available.+ """++ _SUPPORTED_OPS = {"Chunk", "Concatenate", "MergeModulelist", "SplitModulelist"}++ def __init__(self, model, completed_hf_names=None):+ self.model = model+ self.completed_hf_names = completed_hf_names or set()++ def iter_tensors(self) -> Iterator[RestoredTensor]:+ state_dict = self.model.state_dict()+ weight_conversions = self._get_weight_conversions()+ if not weight_conversions:+ for name, tensor in state_dict.items():+ if name in self.completed_hf_names:+ continue+ yield RestoredTensor(name, lambda tensor=tensor: tensor, (name,), "passthrough")+ return++ inverted_transforms = self._get_inverted_transforms(weight_conversions)+ inverted_converters = [transform for transform in inverted_transforms if self._is_weight_converter(transform)]+ inverted_renamings = [+ transform for transform in inverted_transforms if not self._is_weight_converter(transform)+ ]+ pattern_to_converter = {+ pattern: converter for converter in inverted_converters for pattern in converter.source_patterns+ }++ conversion_mapping = {}+ converter_lineage = defaultdict(list)+ converter_fallback_tensors = defaultdict(list)+ converter_order = {}+ ordered_tensors = []++ for source_order, (original_key, tensor) in enumerate(state_dict.items()):+ converter_key, matched_pattern = self._rename_source_key(original_key, [], inverted_converters)+ checkpoint_key, _ = self._rename_source_key(converter_key, inverted_renamings, [])++ if matched_pattern is None:+ if original_key in self.completed_hf_names:+ continue+ transform_kind = "rename" if checkpoint_key != original_key else "passthrough"+ ordered_tensors.append(+ (+ source_order,+ RestoredTensor(+ checkpoint_key,+ lambda tensor=tensor: tensor,+ (original_key,),+ transform_kind,+ ),+ )+ )+ continue++ mapping = conversion_mapping.setdefault(converter_key, deepcopy(pattern_to_converter[matched_pattern]))+ mapping.add_tensor(checkpoint_key, original_key, matched_pattern, tensor)+ converter_lineage[converter_key].append(original_key)+ converter_order.setdefault(converter_key, source_order)+ converter_fallback_tensors[converter_key].append(+ RestoredTensor(checkpoint_key, lambda tensor=tensor: tensor, (original_key,), "passthrough")+ )++ for layer_name, mapping in sorted(conversion_mapping.items()):+ if all(name in self.completed_hf_names for name in converter_lineage[layer_name]):+ continue+ try:+ realized = mapping.convert(layer_name, model=self.model, config=getattr(self.model, "config", None))+ except Exception as exc:+ logger.debug(+ "gguf: falling back to original tensors for failed Transformers weight conversion %s: %s",+ layer_name,+ exc,+ )+ ordered_tensors.extend(+ (converter_order[layer_name], tensor) for tensor in converter_fallback_tensors[layer_name]+ )+ continue+ normalized_realized = []+ for target_name, tensor in sorted(realized.items()):+ if self._is_weight_converter(mapping):+ target_name, _ = self._rename_source_key(target_name, inverted_renamings, [])+ tensor = tensor[0] if isinstance(tensor, list) else tensor+ if tensor is None or tensor.numel() == 0:+ logger.debug(+ "gguf: falling back to original tensors for empty Transformers weight conversion %s",+ layer_name,+ )+ ordered_tensors.extend(+ (converter_order[layer_name], fallback) for fallback in converter_fallback_tensors[layer_name]+ )+ break+ normalized_realized.append((target_name, tensor))+ else:+ lineage = tuple(dict.fromkeys(converter_lineage[layer_name]))+ for target_name, tensor in normalized_realized:+ ordered_tensors.append(+ (+ converter_order[layer_name],+ RestoredTensor(+ target_name,+ lambda tensor=tensor: tensor,+ lineage,+ "converter",+ ),+ )+ )++ for _, restored in sorted(ordered_tensors, key=lambda item: item[0]):+ yield restored++ def _get_weight_conversions(self):+ weight_conversions = getattr(self.model, "_weight_conversions", None)+ if weight_conversions is not None:+ return weight_conversions++ try:+ from transformers.conversion_mapping import get_model_conversion_mapping+ from transformers.core_model_loading import PrefixChange+ except Exception:+ return None++ try:+ conversions = get_model_conversion_mapping(self.model, add_legacy=False)+ except Exception as exc:+ logger.warning("gguf: unable to get Transformers weight conversion mapping: %s", exc)+ return None++ # Match Transformers' own fallback in revert_weight_conversion: if the+ # model was not loaded through from_pretrained, do not invent prefix adds.+ conversions = [conversion for conversion in conversions if not isinstance(conversion, PrefixChange)]+ return conversions or None++ def _get_inverted_transforms(self, weight_conversions):+ inverted = []+ for transform in weight_conversions[::-1]:+ try:+ reversed_transform = transform.reverse_transform()+ except Exception as exc:+ logger.warning("gguf: skipping non-reversible Transformers weight transform %s: %s", transform, exc)+ continue+ if self._is_weight_converter(reversed_transform) and not self._is_supported_converter(reversed_transform):+ logger.warning("gguf: skipping unsupported Transformers weight converter %s", reversed_transform)+ continue+ inverted.append(reversed_transform)+ return inverted++ def _is_supported_converter(self, converter):+ return all(type(op).__name__ in self._SUPPORTED_OPS for op in getattr(converter, "operations", ()))++ @staticmethod+ def _is_weight_converter(transform):+ try:+ from transformers.core_model_loading import WeightConverter+ except Exception:+ return False+ return isinstance(transform, WeightConverter)++ @staticmethod+ def _rename_source_key(source_key, weight_renamings, weight_converters):+ from transformers.core_model_loading import rename_source_key++ return rename_source_key(source_key, weight_renamings, weight_converters)diff --git a/auto_round/export/export_to_gguf/llama_cpp_conversion.py b/auto_round/export/export_to_gguf/llama_cpp_conversion.pyindex 385fc47ec..e057dbbe7 100644--- a/auto_round/export/export_to_gguf/llama_cpp_conversion.py+++ b/auto_round/export/export_to_gguf/llama_cpp_conversion.py@@ -17,6 +17,7 @@ from __future__ import annotationsimport astimport importlibimport json+import loggingimport osimport shutilimport sys@@ -113,11 +114,21 @@ def _clear_loaded_conversion_modules() -> None:del sys.modules[name]+def _configure_conversion_logger() -> None:+ conversion_logger = logging.getLogger("hf-to-gguf")+ conversion_logger.setLevel(logger.level)+ conversion_logger.propagate = False+ for handler in logger.handlers:+ if handler not in conversion_logger.handlers:+ conversion_logger.addHandler(handler)++def _import_conversion_from(root: Path, source: str) -> ConversionContext:global _ACTIVE_CONVERSION_ROOT, _CONVERSION_MODULE, _CONVERSION_SOURCEroot = root.resolve()if _CONVERSION_MODULE is not None and _ACTIVE_CONVERSION_ROOT == root:+ _configure_conversion_logger()return ConversionContext(_CONVERSION_MODULE, _CONVERSION_SOURCE)_clear_loaded_conversion_modules()@@ -133,6 +144,7 @@ def _import_conversion_from(root: Path, source: str) -> ConversionContext:_ACTIVE_CONVERSION_ROOT = root_CONVERSION_MODULE = module_CONVERSION_SOURCE = source+ _configure_conversion_logger()return ConversionContext(module, source)diff --git a/docs/auto_scheme_acc.md b/docs/auto_scheme_acc.mdindex 50d64f90b..ad404c313 100644--- a/docs/auto_scheme_acc.md+++ b/docs/auto_scheme_acc.md@@ -9,7 +9,7 @@ python3 -m auto_round Qwen/Qwen3-8B --options "gguf:q2_k_s,gguf:q4_k_s"eval~~~- llm serve ./test_gguf/Qwen3-8B-w2g16/ --port 8001 --max-model-len 8192 --host 127.0.0.1 --served-model-name qwen+ vllm serve ./test_gguf/Qwen3-8B-w2g16/ --port 8000 --max-model-len 8192 --host 127.0.0.1 --served-model-name qwenevalscope eval --model qwen --api-url http://127.0.0.1:8000/v1 --api-key EMPTY --datasets math_500 gpqa_diamoid mmlu_pro --eval-batch-size 32~~~diff --git a/test/test_cpu/advanced/test_evaluation_functions.py b/test/test_cpu/advanced/test_evaluation_functions.pyindex 9cc99311d..d6476ec21 100644--- a/test/test_cpu/advanced/test_evaluation_functions.py+++ b/test/test_cpu/advanced/test_evaluation_functions.py@@ -25,6 +25,33 @@ from unittest.mock import MagicMock, patchimport pytest+class TestSelectGgufEvalFile:+ """Test GGUF text file selection for evaluation."""++ def test_selects_only_text_gguf_when_mixed_format_filename_differs(self, tmp_path):+ from auto_round.eval.evaluation import select_gguf_eval_file++ (tmp_path / "gemma-4-E4B-it-7.5B-Q2_K_S.gguf").touch()+ (tmp_path / "mmproj-model.gguf").touch()++ gguf_file, candidates = select_gguf_eval_file(str(tmp_path), ["gguf:q2_k_mixed"])++ assert gguf_file == "gemma-4-E4B-it-7.5B-Q2_K_S.gguf"+ assert candidates == ["gemma-4-E4B-it-7.5B-Q2_K_S.gguf"]++ def test_prefers_exact_format_match_when_multiple_text_gguf_files_exist(self, tmp_path):+ from auto_round.eval.evaluation import select_gguf_eval_file++ (tmp_path / "model-Q4_0.gguf").touch()+ (tmp_path / "model-Q8_0.gguf").touch()+ (tmp_path / "mmproj-model.gguf").touch()++ gguf_file, candidates = select_gguf_eval_file(str(tmp_path), ["gguf:q4_0"])++ assert gguf_file == "model-Q4_0.gguf"+ assert candidates == ["model-Q4_0.gguf", "model-Q8_0.gguf"]++class TestParseVllmArgs:"""Test parse_vllm_args function for parsing custom vllm arguments."""diff --git a/test/test_cpu/export/test_gguf_format.py b/test/test_cpu/export/test_gguf_format.pyindex 1e5f54937..d9c04171d 100644--- a/test/test_cpu/export/test_gguf_format.py+++ b/test/test_cpu/export/test_gguf_format.py@@ -201,6 +201,143 @@ class TestGGUF:else:assert abs(file_size - 264) < 5.0+ def test_mmproj_uses_native_f32_export_when_nontext_is_not_quantized(self, tmp_path, monkeypatch):+ from auto_round.export.export_to_gguf import export+ from auto_round.export.export_to_gguf.config import ModelType++ ftypes = {+ "q4_0": object(),+ "f32": object(),+ }+ wrapper_calls = []++ class FakeConfig:+ model_type = "qwen2_vl"++ class FakeModel:+ name_or_path = str(tmp_path)+ config = FakeConfig()++ class FakeModelClass:+ def __init__(self, **kwargs):+ self.__dict__.update(kwargs)+ self.model_arch = "MMPROJ"++ class FakeModelBase:+ @staticmethod+ def load_hparams(_dir_model, _is_mistral_format):+ return {"architectures": ["FakeArchitecture"], "quantization_config": {"bits": 4}}++ class FakeConversion:+ ModelBase = FakeModelBase++ @staticmethod+ def model_type(model_type):+ return model_type++ @staticmethod+ def get_model_architecture(_hparams, _model_type):+ return "FakeArchitecture"++ @staticmethod+ def get_model_class(_model_architecture, model_type):+ return FakeModelClass++ def fake_wrapper(model_instance, **kwargs):+ wrapper_calls.append(model_instance)+ model_instance.was_wrapped = True+ model_instance.model = kwargs["model"]+ return model_instance++ monkeypatch.setattr(export, "FTYPE_MAP", ftypes)+ monkeypatch.setattr(export, "get_conversion", lambda *_args, **_kwargs: FakeConversion)+ monkeypatch.setattr(export, "wrapper_model_instance", fake_wrapper)+ monkeypatch.setattr(export, "handle_special_model", lambda model_instance, _architecture: model_instance)++ model_instance = export.create_model_class(+ output_dir=str(tmp_path),+ model=FakeModel(),+ layer_config={},+ backend="gguf:q4_0",+ model_type=ModelType.MMPROJ,+ quant_nontext_module=False,+ )++ assert wrapper_calls == []+ assert not hasattr(model_instance, "model")+ assert model_instance.ftype is ftypes["f32"]+ assert model_instance.fname_out == tmp_path / "mmproj-model.gguf"++ def test_text_still_uses_autoround_wrapper_when_nontext_is_not_quantized(self, tmp_path, monkeypatch):+ from auto_round.export.export_to_gguf import export+ from auto_round.export.export_to_gguf.config import ModelType++ ftypes = {+ "q4_0": object(),+ "f32": object(),+ }+ wrapper_calls = []++ class FakeConfig:+ model_type = "qwen2_vl"++ class FakeModel:+ name_or_path = str(tmp_path)+ config = FakeConfig()++ class FakeModelClass:+ def __init__(self, **kwargs):+ self.__dict__.update(kwargs)+ self.model_arch = "TEXT"++ class FakeModelBase:+ @staticmethod+ def load_hparams(_dir_model, _is_mistral_format):+ return {"architectures": ["FakeArchitecture"], "quantization_config": {"bits": 4}}++ class FakeConversion:+ ModelBase = FakeModelBase++ @staticmethod+ def model_type(model_type):+ return model_type++ @staticmethod+ def get_model_architecture(_hparams, _model_type):+ return "FakeArchitecture"++ @staticmethod+ def get_model_class(_model_architecture, model_type):+ return FakeModelClass++ def fake_wrapper(model_instance, **kwargs):+ wrapper_calls.append((model_instance, kwargs))+ model_instance.was_wrapped = True+ model_instance.model = kwargs["model"]+ return model_instance++ monkeypatch.setattr(export, "FTYPE_MAP", ftypes)+ monkeypatch.setattr(export, "get_conversion", lambda *_args, **_kwargs: FakeConversion)+ monkeypatch.setattr(export, "wrapper_model_instance", fake_wrapper)+ monkeypatch.setattr(export, "handle_special_model", lambda model_instance, _architecture: model_instance)++ model_instance = export.create_model_class(+ output_dir=str(tmp_path),+ model=FakeModel(),+ layer_config={},+ backend="gguf:q4_0",+ model_type=ModelType.TEXT,+ quant_nontext_module=False,+ )++ assert len(wrapper_calls) == 1+ assert wrapper_calls[0][0] is model_instance+ assert wrapper_calls[0][1]["quant_nontext_module"] is False+ assert model_instance.was_wrapped+ assert isinstance(model_instance.model, FakeModel)+ assert model_instance.ftype is ftypes["q4_0"]+ assert model_instance.fname_out == tmp_path+def test_qtype_setting(self, tiny_qwen_vl_model_path):# Qwen2.5-0.5B-Instruct no output, token_embed q6_k fallbakc to q8_0 336M# Qwen3-0.6B output q6_k, token_embed q4_0 448M@@ -321,8 +458,8 @@ class TestGGUF:gguf_model = GGUFReader(os.path.join(quantized_model_path, gguf_file))assert gguf_model.get_tensor(2).name == "blk.0.attn_k.weight"assert gguf_model.get_tensor(2).tensor_type.name == "Q4_K"- assert gguf_model.get_tensor(9).name == "blk.0.ffn_up_exps.weight"- assert gguf_model.get_tensor(9).tensor_type.name == "Q2_K"+ tensor_types = {tensor.name: tensor.tensor_type.name for tensor in gguf_model.tensors}+ assert tensor_types["blk.0.ffn_up_exps.weight"] == "Q2_K"def test_q2k_mixed_keeps_only_three_dim_expert_weights_at_q2k(self, tiny_qwen_moe_model_path):model_name = tiny_qwen_moe_model_pathdiff --git a/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.pynew file mode 100644index 000000000..d06c1f278--- /dev/null+++ b/test/test_cpu/export/test_gguf_hf_checkpoint_restorer.py@@ -0,0 +1,355 @@+import pytest+import torch+from torch import nn+from transformers.core_model_loading import Chunk, PrefixChange, WeightConverter++from auto_round.export.export_to_gguf.hf_checkpoint_restorer import HFCheckpointRestorer+++class _DummyConfig:+ model_type = "dummy"+++class _DummyModel:+ config = _DummyConfig()++ def __init__(self, state_dict, weight_conversions):+ self._state_dict = state_dict+ self._weight_conversions = weight_conversions++ def state_dict(self):+ return self._state_dict+++def test_restorer_reverses_transformers_prefix_change():+ model = _DummyModel(+ {+ "model.layers.0.mlp.down_proj.weight": torch.ones(2, 2),+ },+ [PrefixChange(prefix_to_remove="language_model", model_prefix="model")],+ )++ restored = list(HFCheckpointRestorer(model).iter_tensors())++ assert [tensor.checkpoint_name for tensor in restored] == [+ "model.language_model.layers.0.mlp.down_proj.weight",+ ]+ assert restored[0].hf_names == ("model.layers.0.mlp.down_proj.weight",)+ assert torch.equal(restored[0].tensor_fn(), torch.ones(2, 2))+++def test_restorer_concats_split_hf_tensors_back_to_checkpoint_tensor():+ model = _DummyModel(+ {+ "model.layers.0.mlp.gate_proj.weight": torch.ones(1, 2),+ "model.layers.0.mlp.up_proj.weight": torch.full((1, 2), 2.0),+ },+ [+ WeightConverter(+ source_patterns="mlp.gate_up_proj.weight",+ target_patterns=["mlp.gate_proj.weight", "mlp.up_proj.weight"],+ operations=[Chunk(dim=0)],+ ),+ ],+ )++ restored = list(HFCheckpointRestorer(model).iter_tensors())++ assert [tensor.checkpoint_name for tensor in restored] == [+ "model.layers.0.mlp.gate_up_proj.weight",+ ]+ assert restored[0].hf_names == (+ "model.layers.0.mlp.gate_proj.weight",+ "model.layers.0.mlp.up_proj.weight",+ )+ assert torch.equal(restored[0].tensor_fn(), torch.tensor([[1.0, 1.0], [2.0, 2.0]]))+++def test_restorer_preserves_model_order_across_converted_and_passthrough_tensors():+ model = _DummyModel(+ {+ "model.layers.0.self_attn.gate_proj.weight": torch.ones(1, 2),+ "model.layers.0.self_attn.up_proj.weight": torch.full((1, 2), 2.0),+ "model.layers.0.mlp.experts.weight": torch.ones(2, 2),+ },+ [+ WeightConverter(+ source_patterns="self_attn.gate_up_proj.weight",+ target_patterns=["self_attn.gate_proj.weight", "self_attn.up_proj.weight"],+ operations=[Chunk(dim=0)],+ ),+ ],+ )++ restored = list(HFCheckpointRestorer(model).iter_tensors())++ assert [tensor.checkpoint_name for tensor in restored] == [+ "model.layers.0.self_attn.gate_up_proj.weight",+ "model.layers.0.mlp.experts.weight",+ ]+++def test_restorer_does_not_rerun_completed_conversion_group(monkeypatch):+ state_dict = {+ "model.layers.0.mlp.gate_proj.weight": torch.ones(1, 2),+ "model.layers.0.mlp.up_proj.weight": torch.full((1, 2), 2.0),+ }+ model = _DummyModel(+ state_dict,+ [+ WeightConverter(+ source_patterns="mlp.gate_up_proj.weight",+ target_patterns=["mlp.gate_proj.weight", "mlp.up_proj.weight"],+ operations=[Chunk(dim=0)],+ ),+ ],+ )++ def fail_if_converted(*args, **kwargs):+ raise AssertionError("completed conversion group was executed again")++ monkeypatch.setattr(WeightConverter, "convert", fail_if_converted)++ restored = list(HFCheckpointRestorer(model, completed_hf_names=set(state_dict)).iter_tensors())++ assert restored == []+++def test_resolve_restored_qtype_uses_consistent_multi_source_layer_config():+ import gguf++ from auto_round.export.export_to_gguf.convert import resolve_restored_qtype++ diagnostics = []+ qtype = resolve_restored_qtype(+ layer_config={+ "model.layers.0.mlp.gate_proj": {"bits": 4, "super_bits": 6, "sym": False},+ "model.layers.0.mlp.up_proj": {"bits": 4, "super_bits": 6, "sym": False},+ },+ hf_names=(+ "model.layers.0.mlp.gate_proj.weight",+ "model.layers.0.mlp.up_proj.weight",+ ),+ checkpoint_name="model.layers.0.mlp.gate_up_proj.weight",+ gguf_name="blk.0.ffn_gate_up.weight",+ fallback_qtype=gguf.GGMLQuantizationType.Q5_K,+ diagnostics=diagnostics,+ )++ assert qtype == gguf.GGMLQuantizationType.Q4_K+ assert diagnostics == []+++def test_resolve_restored_qtype_falls_back_and_records_conflicting_multi_source_layer_config():+ import gguf++ from auto_round.export.export_to_gguf.convert import resolve_restored_qtype++ diagnostics = []+ qtype = resolve_restored_qtype(+ layer_config={+ "model.layers.0.mlp.gate_proj": {"bits": 4, "super_bits": 6, "sym": False},+ "model.layers.0.mlp.up_proj": {"bits": 6, "super_bits": 8, "sym": True},+ },+ hf_names=(+ "model.layers.0.mlp.gate_proj.weight",+ "model.layers.0.mlp.up_proj.weight",+ ),+ checkpoint_name="model.layers.0.mlp.gate_up_proj.weight",+ gguf_name="blk.0.ffn_gate_up.weight",+ fallback_qtype=gguf.GGMLQuantizationType.Q5_K,+ diagnostics=diagnostics,+ )++ assert qtype is None+ assert diagnostics == [+ {+ "checkpoint_name": "model.layers.0.mlp.gate_up_proj.weight",+ "gguf_name": "blk.0.ffn_gate_up.weight",+ "reason": "multi_source_dtype_conflict",+ "fallback_qtype": "Q5_K",+ "hf_sources": [+ {+ "name": "model.layers.0.mlp.gate_proj.weight",+ "layer_config_qtype": "Q4_K",+ },+ {+ "name": "model.layers.0.mlp.up_proj.weight",+ "layer_config_qtype": "Q6_K",+ },+ ],+ }+ ]+++def test_restorer_falls_back_to_original_tensor_when_weight_conversion_fails():+ model = _DummyModel(+ {+ "model.layers.0.mlp.experts.gate_up_proj": torch.tensor([], dtype=torch.bfloat16),+ },+ [+ WeightConverter(+ source_patterns="mlp.experts.gate_up_proj",+ target_patterns=["mlp.experts.*.gate_proj.weight", "mlp.experts.*.up_proj.weight"],+ operations=[Chunk(dim=1)],+ ),+ ],+ )++ restored = list(HFCheckpointRestorer(model).iter_tensors())++ assert [tensor.checkpoint_name for tensor in restored] == ["model.layers.0.mlp.experts.gate_up_proj"]+ assert restored[0].transform_kind == "passthrough"+++def test_get_restored_tensors_replaces_empty_live_tensor_from_checkpoint(monkeypatch):+ from auto_round.export.export_to_gguf import convert++ tensor_name = "model.layers.0.self_attn.q_proj.weight"+ model = _DummyModel({tensor_name: torch.tensor([], dtype=torch.bfloat16)}, [])+ model.tensor_name_list = [tensor_name]++ def iter_checkpoint_tensors(cls):+ if tensor_name not in cls.model.tensor_name_list:+ yield tensor_name, torch.ones(2, 2)++ monkeypatch.setattr(convert, "_iter_extra_tensors", iter_checkpoint_tensors)++ class Converter:+ pass++ converter = Converter()+ converter.model = model+ restored = list(convert.get_restored_tensors(converter))++ assert [tensor.checkpoint_name for tensor in restored] == [tensor_name]+ assert restored[0].hf_names == (tensor_name,)+ assert torch.equal(restored[0].tensor_fn(), torch.ones(2, 2))+++def test_name_resolution_diagnostics_are_never_written_to_export_directory(tmp_path, monkeypatch):+ from auto_round.export.export_to_gguf.convert import _flush_name_resolution_diagnostics++ monkeypatch.setenv("AR_GGUF_DEBUG_NAME_RESOLUTION", "1")++ class Converter:+ fname_out = tmp_path / "model.gguf"+ _gguf_name_resolution_diagnostics = [{"checkpoint_name": "x"}]++ _flush_name_resolution_diagnostics(Converter)++ assert not (tmp_path / "gguf_name_resolution.json").exists()+++def test_gguf_writer_has_tensor_supports_current_gguf_dict_entries():+ import numpy as np+ from gguf.gguf_writer import GGUFWriter++ from auto_round.export.export_to_gguf.convert import _gguf_writer_has_tensor++ writer = GGUFWriter(path=None, arch="llama")+ writer.add_tensor("blk.24.attn_k_norm.weight", np.zeros((1,), dtype=np.float32))++ assert _gguf_writer_has_tensor(writer, "blk.24.attn_k_norm.weight")+ assert not _gguf_writer_has_tensor(writer, "blk.24.attn_v_norm.weight")+++def test_gguf_shape_fallback_updates_layer_config_before_quantization():+ from auto_round.compressors.utils import _apply_gguf_shape_fallback+ from auto_round.export.export_to_gguf.config import GGUF_INNER_CONFIG++ class Model(nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = nn.Linear(1152, 4096, bias=False)++ layer_config = {"proj": {**GGUF_INNER_CONFIG["gguf:q6_k"], "fixed_by_user": True, "scale_dtype": torch.float32}}++ _apply_gguf_shape_fallback(layer_config, Model())++ assert layer_config["proj"]["bits"] == 8+ assert layer_config["proj"]["group_size"] == 32+ assert layer_config["proj"]["super_bits"] is None+ assert layer_config["proj"]["super_group_size"] is None+ assert layer_config["proj"]["fixed_by_user"] is True+ assert layer_config["proj"]["scale_dtype"] is torch.float32+++def test_quant_data_recomputes_scale_when_final_qtype_differs_from_layer_config(monkeypatch):+ import gguf+ import numpy as np++ from auto_round.export.export_to_gguf import convert++ captured = {}++ class Module:+ def __init__(self):+ self.weight = torch.ones(2, 32)+ self.scale = torch.ones(2, 16)++ module = Module()++ class Converter:+ model = object()+ layer_config = {"model.layers.0.self_attn.q_proj": {"bits": 6, "super_bits": 8, "sym": True}}++ def fake_get_module(_model, _layer_name):+ return module++ def fake_ggml_quant(data_torch, ggml_type, device=None, **kwargs):+ captured["data_shape"] = tuple(data_torch.shape)+ captured["ggml_type"] = ggml_type+ captured["scale"] = kwargs["scale"]+ return np.zeros((data_torch.shape[0], data_torch.shape[1]), dtype=np.float32)++ monkeypatch.setattr(convert, "get_module", fake_get_module)+ monkeypatch.setattr(convert, "ggml_quant", fake_ggml_quant)++ data, qtype = convert._quant_data(+ Converter(),+ torch.ones(2, 32),+ gguf.GGMLQuantizationType.Q8_0,+ "model.layers.0.self_attn.q_proj.weight",+ "model.layers.0.self_attn.q_proj.weight",+ "blk.0.attn_q.weight",+ 0,+ device="cpu",+ )++ assert qtype == gguf.GGMLQuantizationType.Q8_0+ assert data.shape == (2, 32)+ assert captured["ggml_type"] == "q8_0"+ assert captured["data_shape"] == (2, 32)+ assert captured["scale"] is None+++def test_quant_data_rejects_layer_scale_that_does_not_match_final_qtype(monkeypatch):+ import gguf++ from auto_round.export.export_to_gguf import convert++ class Module:+ def __init__(self):+ self.weight = torch.ones(2, 32)+ self.scale = torch.ones(2, 2)++ module = Module()++ class Converter:+ model = object()+ layer_config = {"model.layers.0.self_attn.q_proj": {"bits": 8, "group_size": 32, "sym": True}}++ monkeypatch.setattr(convert, "get_module", lambda _model, _layer_name: module)++ with pytest.raises(ValueError, match="q8_0 scale shape"):+ convert._quant_data(+ Converter(),+ torch.ones(2, 32),+ gguf.GGMLQuantizationType.Q8_0,+ "model.layers.0.self_attn.q_proj.weight",+ "model.layers.0.self_attn.q_proj.weight",+ "blk.0.attn_q.weight",+ 0,+ device="cpu",+ )diff --git a/test/test_cuda/export/test_gguf_format.py b/test/test_cuda/export/test_gguf_format.pyindex 458266b86..2c76f0671 100644--- a/test/test_cuda/export/test_gguf_format.py+++ b/test/test_cuda/export/test_gguf_format.py@@ -175,7 +175,7 @@ class TestAutoRound:print(f"{file}: {os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2} MB")file_size = os.path.getsize(os.path.join(quantized_model_path, file)) / 1024**2if "mmproj-model.gguf" in file:- assert abs(file_size - 75) < 5.0+ assert abs(file_size - 62) < 5.0else:assert abs(file_size - 690) < 10.0