PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix few bugs cause less accuracy of gguf format (#1960)
Heng Guoe4404be7ca57
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.pyindex 15f40d0ea..7fba0ca2c 100644--- a/auto_round/algorithms/quantization/base.py+++ b/auto_round/algorithms/quantization/base.py@@ -56,61 +56,52 @@ class RTNLayerFallbackMixin:layer = convert_module_to_hp_if_necessary(layer, self.model_context.amp_dtype, device_manager.device)set_module(self.model, layer_name, layer)tuning_device = layer.tuning_device if hasattr(layer, "tuning_device") else device_manager.device- if (- self.compress_context.is_immediate_packing- and self.compress_context.formats[0].is_gguf()- and not getattr(self.config, "disable_opt_rtn", False)- ):+ try:+ if disable_opt_rtn is None:+ disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))+ if (+ not disable_opt_rtn+ and getattr(self.config, "orig_disable_opt_rtn", None) is None+ and self.model_context.is_moe_model+ and "expert" in layer.global_name+ and "shared_expert" not in layer.global_name+ and self.config.super_bits is None+ ):+ disable_opt_rtn = True+ logger.warning_once(+ "MoE layer detected: optimized RTN is disabled for efficiency. "+ "Use `--enable_opt_rtn` to force-enable it for MoE layers."+ )layer = layer.to(tuning_device)- layer.scale = None- layer.zp = None- else:+ layer = WrapperLinear(+ layer,+ device=tuning_device,+ enable_minmax_tuning=False,+ enable_norm_bias_tuning=False,+ enable_round_tuning=False,+ enable_torch_compile=self.compress_context.enable_torch_compile,+ disable_opt_rtn=disable_opt_rtn,+ iters=0,+ )+ layer = layer.unwrapper({})+ except torch.OutOfMemoryError:+ cuda_error_msg = traceback.format_exc()+ layer = layer.orig_layer if hasattr(layer, "orig_layer") else layertry:- if disable_opt_rtn is None:- disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))- if (- not disable_opt_rtn- and getattr(self.config, "orig_disable_opt_rtn", None) is None- and self.model_context.is_moe_model- and "expert" in layer.global_name- and "shared_expert" not in layer.global_name- and self.config.super_bits is None- ):- disable_opt_rtn = True- logger.warning_once(- "MoE layer detected: optimized RTN is disabled for efficiency. "- "Use `--enable_opt_rtn` to force-enable it for MoE layers."- )- layer = layer.to(tuning_device)+ logger.error(cuda_error_msg)+ logger.warning("falling back to CPU.")+ layer.to("cpu")layer = WrapperLinear(layer,- device=tuning_device,enable_minmax_tuning=False,enable_norm_bias_tuning=False,enable_round_tuning=False,enable_torch_compile=self.compress_context.enable_torch_compile,- disable_opt_rtn=disable_opt_rtn,iters=0,)layer = layer.unwrapper({})- except torch.OutOfMemoryError:- cuda_error_msg = traceback.format_exc()- layer = layer.orig_layer if hasattr(layer, "orig_layer") else layer- try:- logger.error(cuda_error_msg)- logger.warning("falling back to CPU.")- layer.to("cpu")- layer = WrapperLinear(- layer,- enable_minmax_tuning=False,- enable_norm_bias_tuning=False,- enable_round_tuning=False,- enable_torch_compile=self.compress_context.enable_torch_compile,- iters=0,- )- layer = layer.unwrapper({})- except Exception:- raise+ except Exception:+ raiseset_module(self.model, layer_name, layer)self._immediate_pack_and_save_module(layer_name)diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.pyindex dbee03e46..1a6a94a90 100644--- a/auto_round/auto_scheme/delta_loss.py+++ b/auto_round/auto_scheme/delta_loss.py@@ -395,12 +395,12 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):d_wmin=d_wmin,)elif bits in [3, 6]:- group_size = 16- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- scale, d_scale = search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num=1)- tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)qdq_w, _, _ = quant_tensor_gguf_sym_dq(- tensor=tensor, bits=bits, scale_dtype=scale_dtype, imatrix=imatrix, scale=scale, d_scale=d_scale+ tensor=tensor,+ bits=bits,+ scale_dtype=scale_dtype,+ imatrix=imatrix,+ split_num=1,)else:raise ValueError("bits must be in [2,3,4,5,6]")diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex dd7b1dd5b..f1a9a3dfd 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -707,6 +707,20 @@ class BaseCompressor(object):fill_default_value=fill_default_value,gguf_format_name=getattr(self, "_gguf_format_name", None),)+ if self.is_auto_scheme:+ from auto_round.auto_scheme.utils import compute_avg_bits_for_model++ ignore_scale_zp_bits = getattr(self.orig_scheme, "ignore_scale_zp_bits", False)+ avg_bits, total_bits = compute_avg_bits_for_model(+ self.model_context.model,+ ignore_scale_zp_bits=ignore_scale_zp_bits,+ )+ logger.info(+ "AutoScheme final effective avg_bits=%.4f, target avg_bits=%.4f, total_bits=%d",+ avg_bits,+ self.orig_scheme.avg_bits,+ total_bits,+ )# ─────────────────────────────────────────────────────────────────────────@@ -993,58 +1007,68 @@ class BaseCompressor(object):"act_bits","scale_dtype",)- _any_gguf_attr_changed = False- for _attr in _gguf_forwarded_attrs:- if _attr not in self.__dict__:- continue- config_val = getattr(self.quantize_config, _attr, None)- self_val = self.__dict__[_attr]- if _attr not in ("scale_dtype", "act_bits") and config_val != self_val:- _any_gguf_attr_changed = True- if config_val != self_val:- setattr(self.quantize_config, _attr, self_val)- # If format resolution changed scheme attrs, rebuild self.scheme so that- # configure_layer_config() / set_layer_config() see the correct values.- if _any_gguf_attr_changed:- from auto_round.schemes import PRESET_SCHEMES- from auto_round.schemes import QuantizationScheme as _QS-- # Prefer to derive the scheme directly from the gguf format name to- # avoid ambiguity (e.g. Q4_K_S and Q4_K_M share identical weight attrs).- _gguf_preset_scheme = None- _gguf_fmt_name = None- _gguf_original_fmt_name = None- for _fmt in self.formats or []:- # GGUFFormat (outer) has output_format="gguf" but backend.output_format="gguf:q4_k_m"- # GGUFFormat (inner/standalone) has output_format="gguf:q4_k_m"- _of = getattr(_fmt, "output_format", "")- if "gguf" in str(_of):- if str(_of) == "gguf":- # outer GGUFFormat: full format in _original_format (e.g. "gguf:q2_k_mixed")- # or backend.output_format (e.g. "gguf:q2_k_s" after _mixed → _s conversion)- _orig = getattr(_fmt, "_original_format", None)- if _orig:- _gguf_original_fmt_name = str(_orig).upper()- _backend = getattr(_fmt, "backend", None)- _of = getattr(_backend, "output_format", _of) if _backend is not None else _of- _preset_key = str(_of).upper()- if _preset_key in PRESET_SCHEMES:- _gguf_preset_scheme = PRESET_SCHEMES[_preset_key]- _gguf_fmt_name = _preset_key- break- if _gguf_preset_scheme is not None:- # Update scheme on both compressor and quantizer.- self.scheme = _gguf_preset_scheme- # Store the exact gguf format name so configure_layer_config /- # set_layer_config can use it directly, avoiding Q4_K_S / Q4_K_M ambiguity.- self._gguf_format_name = _gguf_fmt_name- # Store original format name (may include _mixed) for _handle_special_schemes- if _gguf_original_fmt_name:- self._gguf_original_format_name = _gguf_original_fmt_name- else:- _new_scheme_dict = {f.name: getattr(self, f.name, None) for f in fields(_QS)}- _new_scheme = _QS.from_dict({k: v for k, v in _new_scheme_dict.items() if v is not None})- self.scheme = _new_scheme+ # Skip this for AutoScheme — the format is for export only, and the+ # per-layer quantization is already determined by orig_scheme.options.+ # Restore scheme attrs from quantize_config for AutoScheme —+ # gguf_args_check may have set __dict__ entries that shadow the proxy.+ if self.is_auto_scheme:+ for _attr in _gguf_forwarded_attrs:+ if _attr in self.__dict__:+ del self.__dict__[_attr]++ if not self.is_auto_scheme:+ _any_gguf_attr_changed = False+ for _attr in _gguf_forwarded_attrs:+ if _attr not in self.__dict__:+ continue+ config_val = getattr(self.quantize_config, _attr, None)+ self_val = self.__dict__[_attr]+ if _attr not in ("scale_dtype", "act_bits") and config_val != self_val:+ _any_gguf_attr_changed = True+ if config_val != self_val:+ setattr(self.quantize_config, _attr, self_val)+ # If format resolution changed scheme attrs, rebuild self.scheme so that+ # configure_layer_config() / set_layer_config() see the correct values.+ if _any_gguf_attr_changed:+ from auto_round.schemes import PRESET_SCHEMES+ from auto_round.schemes import QuantizationScheme as _QS++ # Prefer to derive the scheme directly from the gguf format name to+ # avoid ambiguity (e.g. Q4_K_S and Q4_K_M share identical weight attrs).+ _gguf_preset_scheme = None+ _gguf_fmt_name = None+ _gguf_original_fmt_name = None+ for _fmt in self.formats or []:+ # GGUFFormat (outer) has output_format="gguf" but backend.output_format="gguf:q4_k_m"+ # GGUFFormat (inner/standalone) has output_format="gguf:q4_k_m"+ _of = getattr(_fmt, "output_format", "")+ if "gguf" in str(_of):+ if str(_of) == "gguf":+ # outer GGUFFormat: full format in _original_format (e.g. "gguf:q2_k_mixed")+ # or backend.output_format (e.g. "gguf:q2_k_s" after _mixed → _s conversion)+ _orig = getattr(_fmt, "_original_format", None)+ if _orig:+ _gguf_original_fmt_name = str(_orig).upper()+ _backend = getattr(_fmt, "backend", None)+ _of = getattr(_backend, "output_format", _of) if _backend is not None else _of+ _preset_key = str(_of).upper()+ if _preset_key in PRESET_SCHEMES:+ _gguf_preset_scheme = PRESET_SCHEMES[_preset_key]+ _gguf_fmt_name = _preset_key+ break+ if _gguf_preset_scheme is not None:+ # Update scheme on both compressor and quantizer.+ self.scheme = _gguf_preset_scheme+ # Store the exact gguf format name so configure_layer_config /+ # set_layer_config can use it directly, avoiding Q4_K_S / Q4_K_M ambiguity.+ self._gguf_format_name = _gguf_fmt_name+ # Store original format name (may include _mixed) for _handle_special_schemes+ if _gguf_original_fmt_name:+ self._gguf_original_format_name = _gguf_original_fmt_name+ else:+ _new_scheme_dict = {f.name: getattr(self, f.name, None) for f in fields(_QS)}+ _new_scheme = _QS.from_dict({k: v for k, v in _new_scheme_dict.items() if v is not None})+ self.scheme = _new_scheme_gguf_layer_cfg = {k: v for k, v in (self.__dict__.get("layer_config") or {}).items() if k not in (_pre_gguf_layer_config)diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 7fea96c9a..28a407cd0 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -17,7 +17,7 @@ import torchfrom auto_round.data_type.register import register_dtypefrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste-from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES+from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_Kfrom auto_round.export.export_to_gguf.packing import make_q3_quants, make_qx_quants, make_qx_quants_chunkfrom auto_round.logger import loggerfrom auto_round.utils import get_reciprocal@@ -462,10 +462,8 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatri}weights = imatrix.reshape(1, -1)-weights = weights.expand(tensor.numel() // weights.numel(), -1)quant_weights = weights.reshape(tensor.shape)-quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits)# sigma2 = torch.sum(torch.pow(tensor, 2), dim=-1, keepdim=True) / QK_K@@ -789,7 +787,7 @@ def quant_tensor_gguf_sym_dq(Quantized and de-quantized tensor, scale, zero-point"""- from auto_round.export.export_to_gguf.config import K_SCALE_SIZE, QK_K+ from auto_round.export.export_to_gguf.config import K_SCALE_SIZEif bits not in [3, 6]:raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex be21e33e4..3c40d18be 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -591,7 +591,10 @@ def _flush_name_resolution_diagnostics(cls):def prepare_tensors(cls):device = get_packing_device(cls.device)if not hasattr(cls, "_gguf_dtype_selector"):- cls._gguf_dtype_selector = GGUFDTypeSelector(cls.hparams, cls.ftype, cls.model_arch)+ has_tied_embeddings = cls.hparams.get("tie_word_embeddings", False)+ cls._gguf_dtype_selector = GGUFDTypeSelector(+ cls.hparams, cls.ftype, cls.model_arch, has_tied_embeddings=has_tied_embeddings+ )cls._gguf_dtype_selector.n_attention_wv = _count_attention_wv_tensors(cls)if not hasattr(cls, "_gguf_name_resolution_diagnostics"):cls._gguf_name_resolution_diagnostics = []diff --git a/auto_round/export/export_to_gguf/gguf_dtype.py b/auto_round/export/export_to_gguf/gguf_dtype.pyindex 93c9f2ff9..1817ad46d 100644--- a/auto_round/export/export_to_gguf/gguf_dtype.py+++ b/auto_round/export/export_to_gguf/gguf_dtype.py@@ -177,6 +177,7 @@ class GGUFDTypeSelector:n_layer: int | None = None,n_attention_wv: int | None = None,has_imatrix: bool = False,+ has_tied_embeddings: bool = False,):self.hparams = hparamsself.ftype = ftype@@ -184,6 +185,7 @@ class GGUFDTypeSelector:self.n_layer = n_layerself.n_attention_wv = n_attention_wvself.has_imatrix = has_imatrix+ self.has_tied_embeddings = has_tied_embeddingsself.i_attention_wv = 0self.i_ffn_down = 0@@ -222,6 +224,20 @@ class GGUFDTypeSelector:or 0)+ # llama.cpp: output & token_embd handling (llama_tensor_get_type outer wrapper)+ if category == TensorCategory.OUTPUT or (self.has_tied_embeddings and category == TensorCategory.TOKEN_EMBD):+ # llama.cpp: if new_type != Q8_0, upgrade to Q6_K+ # Skip F32/F16/BF16 (non-quantized types that won't reach here in practice)+ if qtype not in (+ gguf.GGMLQuantizationType.F32,+ gguf.GGMLQuantizationType.F16,+ gguf.GGMLQuantizationType.BF16,+ gguf.GGMLQuantizationType.Q8_0,+ ):+ qtype = gguf.GGMLQuantizationType.Q6_K+ elif category == TensorCategory.TOKEN_EMBD:+ pass+if _is_attn_v_like(category):if self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:qtype = gguf.GGMLQuantizationType.Q4_K if n_gqa >= 4 else gguf.GGMLQuantizationType.Q3_Kdiff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex bff88da2f..fa0210868 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -83,6 +83,8 @@ def ggml_quant(return t# Q4_0 / Q5_0 etc. compute their own scales; metadata from a different# group size (e.g. 256 vs block_size 32) cannot be reshaped and is unused.+ if t.dim() == 1: # imatrix / per-channel importance vector; keep as-is+ return t.to(device)if t.nelement() % blocks.shape[0] != 0:return Nonereturn t.to(device).reshape(blocks.shape[0], -1)@@ -768,9 +770,14 @@ def q4_k_quant_block(output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)q_scales = (scales * get_reciprocal(output_d)).round_().clamp_(0, 63).to(torch.uint8)q_mins = (mins * get_reciprocal(output_dmin)).round_().clamp_(0, 63).to(torch.uint8)+ output_d = output_d.to(torch.float16).to(torch.float32)+ output_dmin = output_dmin.to(torch.float16).to(torch.float32)+ d_tmp = output_d * q_scales.to(torch.float32)+ dm_tmp = output_dmin * q_mins.to(torch.float32)+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))all_L = (- blocks.add_(mins.unsqueeze(-1))- .mul_(get_reciprocal(scales.unsqueeze(-1)))+ blocks.add_(dm_tmp.unsqueeze(-1))+ .mul_(get_reciprocal(d_tmp.unsqueeze(-1))).round_().clamp_(0, 15).to(torch.uint8)@@ -814,9 +821,14 @@ def q4_k_quant_block(output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)q_scales = (scales * get_reciprocal(output_d)).round_().clamp_(0, 63).to(torch.uint8)q_mins = (mins * get_reciprocal(output_dmin)).round_().clamp_(0, 63).to(torch.uint8)+ output_d = output_d.to(torch.float16).to(torch.float32)+ output_dmin = output_dmin.to(torch.float16).to(torch.float32)+ d_tmp = output_d * q_scales.to(torch.float32)+ dm_tmp = output_dmin * q_mins.to(torch.float32)+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))all_L = (- blocks.add_(mins.unsqueeze(-1))- .mul_(get_reciprocal(scales.unsqueeze(-1)))+ blocks.add_(dm_tmp.unsqueeze(-1))+ .mul_(get_reciprocal(d_tmp.unsqueeze(-1))).round_().clamp_(0, 15).to(torch.uint8)@@ -864,9 +876,14 @@ def q5_k_quant_block(output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)q_scales = (scales * get_reciprocal(output_d)).round_().clamp_(0, 63).to(torch.uint8)q_mins = (mins * get_reciprocal(output_dmin)).round_().clamp_(0, 63).to(torch.uint8)+ output_d = output_d.to(torch.float16).to(torch.float32)+ output_dmin = output_dmin.to(torch.float16).to(torch.float32)+ d_tmp = output_d * q_scales.to(torch.float32)+ dm_tmp = output_dmin * q_mins.to(torch.float32)+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))all_L = (- blocks.add_(mins.unsqueeze(-1))- .mul_(get_reciprocal(scales.unsqueeze(-1)))+ blocks.add_(dm_tmp.unsqueeze(-1))+ .mul_(get_reciprocal(d_tmp.unsqueeze(-1))).round_().clamp_(0, 31).to(torch.uint8)@@ -909,9 +926,14 @@ def q5_k_quant_block(output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)q_scales = (scales * get_reciprocal(output_d)).round_().clamp_(0, 63).to(torch.uint8)q_mins = (mins * get_reciprocal(output_dmin)).round_().clamp_(0, 63).to(torch.uint8)+ output_d = output_d.to(torch.float16).to(torch.float32)+ output_dmin = output_dmin.to(torch.float16).to(torch.float32)+ d_tmp = output_d * q_scales.to(torch.float32)+ dm_tmp = output_dmin * q_mins.to(torch.float32)+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))all_L = (- blocks.add_(mins.unsqueeze(-1))- .mul_(get_reciprocal(scales.unsqueeze(-1)))+ blocks.add_(dm_tmp.unsqueeze(-1))+ .mul_(get_reciprocal(d_tmp.unsqueeze(-1))).round_().clamp_(0, 31).to(torch.uint8)@@ -952,8 +974,10 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,output_d = d_scale.reshape(-1, 1).to(torch.float32)rd = get_reciprocal(output_d)output_scale = scales.mul(rd).round_().clamp_(max=127).to(torch.int8)- rs = get_reciprocal(scales).unsqueeze_(-1) # unsqueeze for broadcasting- all_L = blocks.mul_(rs).add_(32).round_().clamp_(0, 63).to(torch.uint8)+ output_d = output_d.to(torch.float16).to(torch.float32)+ d_tmp = output_d * output_scale.to(torch.float32)+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))+ all_L = blocks.mul_(get_reciprocal(d_tmp).unsqueeze_(-1)).add_(32).round_().clamp_(0, 63).to(torch.uint8)elif original:scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)imax = abs(scales).argmax(dim=-1, keepdim=True)@@ -978,7 +1002,10 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,scales = scales.reshape((-1, QK_K // 16))output_d = d_scale.reshape(-1, 1).to(torch.float32)output_scale = (scales * get_reciprocal(output_d)).round_().clamp_(max=127).to(torch.int8)- all_L = blocks.mul_(get_reciprocal(scales.unsqueeze(-1))).add_(32).round_().clamp_(0, 63).to(torch.uint8)+ output_d = output_d.to(torch.float16).to(torch.float32)+ d_tmp = output_d * output_scale.to(torch.float32)+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))+ all_L = blocks.mul_(get_reciprocal(d_tmp).unsqueeze_(-1)).add_(32).round_().clamp_(0, 63).to(torch.uint8)tmp_L = all_L.reshape(nb, 4, 64) & 0xFoutput_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)diff --git a/auto_round/formats.py b/auto_round/formats.pyindex 5dcd3ca6b..d16de4756 100644--- a/auto_round/formats.py+++ b/auto_round/formats.py@@ -95,12 +95,23 @@ def _check_compatibility(formats: list[str], ar: BaseCompressor):if gguf_format_name.lower().endswith("mixed"):gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")if any([f.lower() not in ["fake", gguf_format_name.lower()] for f in formats]):- tmp_format_name = gguf_format_name.lower() if "fake" not in formats else f"{gguf_format_name.lower()},fake"- logger.warning(- f"reset format {','.join(formats)} to {tmp_format_name} "- f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()} or fake"- )- formats = tmp_format_name.split(",")+ has_gguf_format = any(f.lower().startswith("gguf") for f in formats if f.lower() != "fake")+ if has_gguf_format:+ logger.warning(+ f"scheme {gguf_format_name} is GGUF, but format {','.join(formats)} specifies "+ f"a different GGUF type. The scheme-driven per-layer quantization may differ from the "+ f"file-level GGUF format type."+ )+ else:+ tmp_format_name = (+ gguf_format_name.lower() if "fake" not in formats else (f"{gguf_format_name.lower()},fake")+ )+ logger.warning(+ f"reset format {','.join(formats)} to {tmp_format_name} "+ f"since scheme {gguf_format_name} can only be exported to format "+ f"{gguf_format_name.lower()} or fake"+ )+ formats = tmp_format_name.split(",")if isinstance(ar.group_size, tuple) and any(["auto_round" in f.lower() for f in formats]):logger.warning(@@ -149,6 +160,11 @@ def get_formats(formats = [fmt for fmt in formats if fmt is not None]+ # Ensure fake format is processed before GGUF — GGUF export may clear+ # model weights via low_cpu_mem_usage, causing zeroed weights for fake.+ if any(fmt.is_gguf() for fmt in formats) and any(fmt.is_fake() for fmt in formats):+ formats.sort(key=lambda f: 0 if f.is_fake() else 1)+if len(formats) == 1 and formats[0].is_gguf() and ar.scale_dtype != torch.float32:ar.scale_dtype = torch.float32logger.info("change `scale_dtype` to `torch.float32` for gguf format")@@ -795,7 +811,7 @@ class GGUFFormat(OutputFormat):gguf_format, ar.layer_config, ar.model, quant_nontext_module=ar.quant_nontext_module)gguf_format = gguf_format.lower().replace("_mixed", "_s")- if isinstance(scheme, str) and scheme.lower() != gguf_format:+ if isinstance(scheme, str) and scheme.lower() != gguf_format and not getattr(ar, "is_auto_scheme", False):logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")ar.scheme = gguf_formatself.output_format = gguf_format