Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix few bugs cause less accuracy of gguf format (#1960)

Heng Guoe4404be7ca57
diff --git a/auto_round/algorithms/quantization/base.py b/auto_round/algorithms/quantization/base.py
index 15f40d0ea..7fba0ca2c 100644
--- a/auto_round/algorithms/quantization/base.py
+++ b/auto_round/algorithms/quantization/base.py
@@ -56,61 +56,52 @@ class RTNLayerFallbackMixin:
layer = convert_module_to_hp_if_necessary(layer, self.model_context.amp_dtype, device_manager.device)
set_module(self.model, layer_name, layer)
tuning_device = layer.tuning_device if hasattr(layer, "tuning_device") else device_manager.device
- if (
- self.compress_context.is_immediate_packing
- and self.compress_context.formats[0].is_gguf()
- and not getattr(self.config, "disable_opt_rtn", False)
- ):
+ try:
+ if disable_opt_rtn is None:
+ disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))
+ if (
+ not disable_opt_rtn
+ and getattr(self.config, "orig_disable_opt_rtn", None) is None
+ and self.model_context.is_moe_model
+ and "expert" in layer.global_name
+ and "shared_expert" not in layer.global_name
+ and self.config.super_bits is None
+ ):
+ disable_opt_rtn = True
+ logger.warning_once(
+ "MoE layer detected: optimized RTN is disabled for efficiency. "
+ "Use `--enable_opt_rtn` to force-enable it for MoE layers."
+ )
layer = layer.to(tuning_device)
- layer.scale = None
- layer.zp = None
- else:
+ layer = WrapperLinear(
+ layer,
+ device=tuning_device,
+ enable_minmax_tuning=False,
+ enable_norm_bias_tuning=False,
+ enable_round_tuning=False,
+ enable_torch_compile=self.compress_context.enable_torch_compile,
+ disable_opt_rtn=disable_opt_rtn,
+ iters=0,
+ )
+ layer = layer.unwrapper({})
+ except torch.OutOfMemoryError:
+ cuda_error_msg = traceback.format_exc()
+ layer = layer.orig_layer if hasattr(layer, "orig_layer") else layer
try:
- if disable_opt_rtn is None:
- disable_opt_rtn = bool(getattr(self.config, "disable_opt_rtn", False))
- if (
- not disable_opt_rtn
- and getattr(self.config, "orig_disable_opt_rtn", None) is None
- and self.model_context.is_moe_model
- and "expert" in layer.global_name
- and "shared_expert" not in layer.global_name
- and self.config.super_bits is None
- ):
- disable_opt_rtn = True
- logger.warning_once(
- "MoE layer detected: optimized RTN is disabled for efficiency. "
- "Use `--enable_opt_rtn` to force-enable it for MoE layers."
- )
- layer = layer.to(tuning_device)
+ logger.error(cuda_error_msg)
+ logger.warning("falling back to CPU.")
+ layer.to("cpu")
layer = WrapperLinear(
layer,
- device=tuning_device,
enable_minmax_tuning=False,
enable_norm_bias_tuning=False,
enable_round_tuning=False,
enable_torch_compile=self.compress_context.enable_torch_compile,
- disable_opt_rtn=disable_opt_rtn,
iters=0,
)
layer = layer.unwrapper({})
- except torch.OutOfMemoryError:
- cuda_error_msg = traceback.format_exc()
- layer = layer.orig_layer if hasattr(layer, "orig_layer") else layer
- try:
- logger.error(cuda_error_msg)
- logger.warning("falling back to CPU.")
- layer.to("cpu")
- layer = WrapperLinear(
- layer,
- enable_minmax_tuning=False,
- enable_norm_bias_tuning=False,
- enable_round_tuning=False,
- enable_torch_compile=self.compress_context.enable_torch_compile,
- iters=0,
- )
- layer = layer.unwrapper({})
- except Exception:
- raise
+ except Exception:
+ raise
set_module(self.model, layer_name, layer)
self._immediate_pack_and_save_module(layer_name)
diff --git a/auto_round/auto_scheme/delta_loss.py b/auto_round/auto_scheme/delta_loss.py
index dbee03e46..1a6a94a90 100644
--- a/auto_round/auto_scheme/delta_loss.py
+++ b/auto_round/auto_scheme/delta_loss.py
@@ -395,12 +395,12 @@ class AutoSchemeWrapperLinearForGGUFKImatrix(AutoSchemeWrapperLinear):
d_wmin=d_wmin,
)
elif bits in [3, 6]:
- group_size = 16
- tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
- scale, d_scale = search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num=1)
- tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)
qdq_w, _, _ = quant_tensor_gguf_sym_dq(
- tensor=tensor, bits=bits, scale_dtype=scale_dtype, imatrix=imatrix, scale=scale, d_scale=d_scale
+ tensor=tensor,
+ bits=bits,
+ scale_dtype=scale_dtype,
+ imatrix=imatrix,
+ split_num=1,
)
else:
raise ValueError("bits must be in [2,3,4,5,6]")
diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.py
index dd7b1dd5b..f1a9a3dfd 100644
--- a/auto_round/compressors/base.py
+++ b/auto_round/compressors/base.py
@@ -707,6 +707,20 @@ class BaseCompressor(object):
fill_default_value=fill_default_value,
gguf_format_name=getattr(self, "_gguf_format_name", None),
)
+ if self.is_auto_scheme:
+ from auto_round.auto_scheme.utils import compute_avg_bits_for_model
+
+ ignore_scale_zp_bits = getattr(self.orig_scheme, "ignore_scale_zp_bits", False)
+ avg_bits, total_bits = compute_avg_bits_for_model(
+ self.model_context.model,
+ ignore_scale_zp_bits=ignore_scale_zp_bits,
+ )
+ logger.info(
+ "AutoScheme final effective avg_bits=%.4f, target avg_bits=%.4f, total_bits=%d",
+ avg_bits,
+ self.orig_scheme.avg_bits,
+ total_bits,
+ )
# ─────────────────────────────────────────────────────────────────────────
@@ -993,58 +1007,68 @@ class BaseCompressor(object):
"act_bits",
"scale_dtype",
)
- _any_gguf_attr_changed = False
- for _attr in _gguf_forwarded_attrs:
- if _attr not in self.__dict__:
- continue
- config_val = getattr(self.quantize_config, _attr, None)
- self_val = self.__dict__[_attr]
- if _attr not in ("scale_dtype", "act_bits") and config_val != self_val:
- _any_gguf_attr_changed = True
- if config_val != self_val:
- setattr(self.quantize_config, _attr, self_val)
- # If format resolution changed scheme attrs, rebuild self.scheme so that
- # configure_layer_config() / set_layer_config() see the correct values.
- if _any_gguf_attr_changed:
- from auto_round.schemes import PRESET_SCHEMES
- from auto_round.schemes import QuantizationScheme as _QS
-
- # Prefer to derive the scheme directly from the gguf format name to
- # avoid ambiguity (e.g. Q4_K_S and Q4_K_M share identical weight attrs).
- _gguf_preset_scheme = None
- _gguf_fmt_name = None
- _gguf_original_fmt_name = None
- for _fmt in self.formats or []:
- # GGUFFormat (outer) has output_format="gguf" but backend.output_format="gguf:q4_k_m"
- # GGUFFormat (inner/standalone) has output_format="gguf:q4_k_m"
- _of = getattr(_fmt, "output_format", "")
- if "gguf" in str(_of):
- if str(_of) == "gguf":
- # outer GGUFFormat: full format in _original_format (e.g. "gguf:q2_k_mixed")
- # or backend.output_format (e.g. "gguf:q2_k_s" after _mixed → _s conversion)
- _orig = getattr(_fmt, "_original_format", None)
- if _orig:
- _gguf_original_fmt_name = str(_orig).upper()
- _backend = getattr(_fmt, "backend", None)
- _of = getattr(_backend, "output_format", _of) if _backend is not None else _of
- _preset_key = str(_of).upper()
- if _preset_key in PRESET_SCHEMES:
- _gguf_preset_scheme = PRESET_SCHEMES[_preset_key]
- _gguf_fmt_name = _preset_key
- break
- if _gguf_preset_scheme is not None:
- # Update scheme on both compressor and quantizer.
- self.scheme = _gguf_preset_scheme
- # Store the exact gguf format name so configure_layer_config /
- # set_layer_config can use it directly, avoiding Q4_K_S / Q4_K_M ambiguity.
- self._gguf_format_name = _gguf_fmt_name
- # Store original format name (may include _mixed) for _handle_special_schemes
- if _gguf_original_fmt_name:
- self._gguf_original_format_name = _gguf_original_fmt_name
- else:
- _new_scheme_dict = {f.name: getattr(self, f.name, None) for f in fields(_QS)}
- _new_scheme = _QS.from_dict({k: v for k, v in _new_scheme_dict.items() if v is not None})
- self.scheme = _new_scheme
+ # Skip this for AutoScheme — the format is for export only, and the
+ # per-layer quantization is already determined by orig_scheme.options.
+ # Restore scheme attrs from quantize_config for AutoScheme —
+ # gguf_args_check may have set __dict__ entries that shadow the proxy.
+ if self.is_auto_scheme:
+ for _attr in _gguf_forwarded_attrs:
+ if _attr in self.__dict__:
+ del self.__dict__[_attr]
+
+ if not self.is_auto_scheme:
+ _any_gguf_attr_changed = False
+ for _attr in _gguf_forwarded_attrs:
+ if _attr not in self.__dict__:
+ continue
+ config_val = getattr(self.quantize_config, _attr, None)
+ self_val = self.__dict__[_attr]
+ if _attr not in ("scale_dtype", "act_bits") and config_val != self_val:
+ _any_gguf_attr_changed = True
+ if config_val != self_val:
+ setattr(self.quantize_config, _attr, self_val)
+ # If format resolution changed scheme attrs, rebuild self.scheme so that
+ # configure_layer_config() / set_layer_config() see the correct values.
+ if _any_gguf_attr_changed:
+ from auto_round.schemes import PRESET_SCHEMES
+ from auto_round.schemes import QuantizationScheme as _QS
+
+ # Prefer to derive the scheme directly from the gguf format name to
+ # avoid ambiguity (e.g. Q4_K_S and Q4_K_M share identical weight attrs).
+ _gguf_preset_scheme = None
+ _gguf_fmt_name = None
+ _gguf_original_fmt_name = None
+ for _fmt in self.formats or []:
+ # GGUFFormat (outer) has output_format="gguf" but backend.output_format="gguf:q4_k_m"
+ # GGUFFormat (inner/standalone) has output_format="gguf:q4_k_m"
+ _of = getattr(_fmt, "output_format", "")
+ if "gguf" in str(_of):
+ if str(_of) == "gguf":
+ # outer GGUFFormat: full format in _original_format (e.g. "gguf:q2_k_mixed")
+ # or backend.output_format (e.g. "gguf:q2_k_s" after _mixed → _s conversion)
+ _orig = getattr(_fmt, "_original_format", None)
+ if _orig:
+ _gguf_original_fmt_name = str(_orig).upper()
+ _backend = getattr(_fmt, "backend", None)
+ _of = getattr(_backend, "output_format", _of) if _backend is not None else _of
+ _preset_key = str(_of).upper()
+ if _preset_key in PRESET_SCHEMES:
+ _gguf_preset_scheme = PRESET_SCHEMES[_preset_key]
+ _gguf_fmt_name = _preset_key
+ break
+ if _gguf_preset_scheme is not None:
+ # Update scheme on both compressor and quantizer.
+ self.scheme = _gguf_preset_scheme
+ # Store the exact gguf format name so configure_layer_config /
+ # set_layer_config can use it directly, avoiding Q4_K_S / Q4_K_M ambiguity.
+ self._gguf_format_name = _gguf_fmt_name
+ # Store original format name (may include _mixed) for _handle_special_schemes
+ if _gguf_original_fmt_name:
+ self._gguf_original_format_name = _gguf_original_fmt_name
+ else:
+ _new_scheme_dict = {f.name: getattr(self, f.name, None) for f in fields(_QS)}
+ _new_scheme = _QS.from_dict({k: v for k, v in _new_scheme_dict.items() if v is not None})
+ self.scheme = _new_scheme
_gguf_layer_cfg = {
k: v for k, v in (self.__dict__.get("layer_config") or {}).items() if k not in (_pre_gguf_layer_config)
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 7fea96c9a..28a407cd0 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -17,7 +17,7 @@ import torch
from auto_round.data_type.register import register_dtype
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste
-from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES
+from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
from auto_round.export.export_to_gguf.packing import make_q3_quants, make_qx_quants, make_qx_quants_chunk
from auto_round.logger import logger
from auto_round.utils import get_reciprocal
@@ -462,10 +462,8 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatri
}
weights = imatrix.reshape(1, -1)
-
weights = weights.expand(tensor.numel() // weights.numel(), -1)
quant_weights = weights.reshape(tensor.shape)
-
quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits)
# sigma2 = torch.sum(torch.pow(tensor, 2), dim=-1, keepdim=True) / QK_K
@@ -789,7 +787,7 @@ def quant_tensor_gguf_sym_dq(
Quantized and de-quantized tensor, scale, zero-point
"""
- from auto_round.export.export_to_gguf.config import K_SCALE_SIZE, QK_K
+ from auto_round.export.export_to_gguf.config import K_SCALE_SIZE
if bits not in [3, 6]:
raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index be21e33e4..3c40d18be 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -591,7 +591,10 @@ def _flush_name_resolution_diagnostics(cls):
def prepare_tensors(cls):
device = get_packing_device(cls.device)
if not hasattr(cls, "_gguf_dtype_selector"):
- cls._gguf_dtype_selector = GGUFDTypeSelector(cls.hparams, cls.ftype, cls.model_arch)
+ has_tied_embeddings = cls.hparams.get("tie_word_embeddings", False)
+ cls._gguf_dtype_selector = GGUFDTypeSelector(
+ cls.hparams, cls.ftype, cls.model_arch, has_tied_embeddings=has_tied_embeddings
+ )
cls._gguf_dtype_selector.n_attention_wv = _count_attention_wv_tensors(cls)
if not hasattr(cls, "_gguf_name_resolution_diagnostics"):
cls._gguf_name_resolution_diagnostics = []
diff --git a/auto_round/export/export_to_gguf/gguf_dtype.py b/auto_round/export/export_to_gguf/gguf_dtype.py
index 93c9f2ff9..1817ad46d 100644
--- a/auto_round/export/export_to_gguf/gguf_dtype.py
+++ b/auto_round/export/export_to_gguf/gguf_dtype.py
@@ -177,6 +177,7 @@ class GGUFDTypeSelector:
n_layer: int | None = None,
n_attention_wv: int | None = None,
has_imatrix: bool = False,
+ has_tied_embeddings: bool = False,
):
self.hparams = hparams
self.ftype = ftype
@@ -184,6 +185,7 @@ class GGUFDTypeSelector:
self.n_layer = n_layer
self.n_attention_wv = n_attention_wv
self.has_imatrix = has_imatrix
+ self.has_tied_embeddings = has_tied_embeddings
self.i_attention_wv = 0
self.i_ffn_down = 0
@@ -222,6 +224,20 @@ class GGUFDTypeSelector:
or 0
)
+ # llama.cpp: output & token_embd handling (llama_tensor_get_type outer wrapper)
+ if category == TensorCategory.OUTPUT or (self.has_tied_embeddings and category == TensorCategory.TOKEN_EMBD):
+ # llama.cpp: if new_type != Q8_0, upgrade to Q6_K
+ # Skip F32/F16/BF16 (non-quantized types that won't reach here in practice)
+ if qtype not in (
+ gguf.GGMLQuantizationType.F32,
+ gguf.GGMLQuantizationType.F16,
+ gguf.GGMLQuantizationType.BF16,
+ gguf.GGMLQuantizationType.Q8_0,
+ ):
+ qtype = gguf.GGMLQuantizationType.Q6_K
+ elif category == TensorCategory.TOKEN_EMBD:
+ pass
+
if _is_attn_v_like(category):
if self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K:
qtype = gguf.GGMLQuantizationType.Q4_K if n_gqa >= 4 else gguf.GGMLQuantizationType.Q3_K
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index bff88da2f..fa0210868 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -83,6 +83,8 @@ def ggml_quant(
return t
# Q4_0 / Q5_0 etc. compute their own scales; metadata from a different
# group size (e.g. 256 vs block_size 32) cannot be reshaped and is unused.
+ if t.dim() == 1: # imatrix / per-channel importance vector; keep as-is
+ return t.to(device)
if t.nelement() % blocks.shape[0] != 0:
return None
return t.to(device).reshape(blocks.shape[0], -1)
@@ -768,9 +770,14 @@ def q4_k_quant_block(
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
q_scales = (scales * get_reciprocal(output_d)).round_().clamp_(0, 63).to(torch.uint8)
q_mins = (mins * get_reciprocal(output_dmin)).round_().clamp_(0, 63).to(torch.uint8)
+ output_d = output_d.to(torch.float16).to(torch.float32)
+ output_dmin = output_dmin.to(torch.float16).to(torch.float32)
+ d_tmp = output_d * q_scales.to(torch.float32)
+ dm_tmp = output_dmin * q_mins.to(torch.float32)
+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))
all_L = (
- blocks.add_(mins.unsqueeze(-1))
- .mul_(get_reciprocal(scales.unsqueeze(-1)))
+ blocks.add_(dm_tmp.unsqueeze(-1))
+ .mul_(get_reciprocal(d_tmp.unsqueeze(-1)))
.round_()
.clamp_(0, 15)
.to(torch.uint8)
@@ -814,9 +821,14 @@ def q4_k_quant_block(
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
q_scales = (scales * get_reciprocal(output_d)).round_().clamp_(0, 63).to(torch.uint8)
q_mins = (mins * get_reciprocal(output_dmin)).round_().clamp_(0, 63).to(torch.uint8)
+ output_d = output_d.to(torch.float16).to(torch.float32)
+ output_dmin = output_dmin.to(torch.float16).to(torch.float32)
+ d_tmp = output_d * q_scales.to(torch.float32)
+ dm_tmp = output_dmin * q_mins.to(torch.float32)
+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))
all_L = (
- blocks.add_(mins.unsqueeze(-1))
- .mul_(get_reciprocal(scales.unsqueeze(-1)))
+ blocks.add_(dm_tmp.unsqueeze(-1))
+ .mul_(get_reciprocal(d_tmp.unsqueeze(-1)))
.round_()
.clamp_(0, 15)
.to(torch.uint8)
@@ -864,9 +876,14 @@ def q5_k_quant_block(
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
q_scales = (scales * get_reciprocal(output_d)).round_().clamp_(0, 63).to(torch.uint8)
q_mins = (mins * get_reciprocal(output_dmin)).round_().clamp_(0, 63).to(torch.uint8)
+ output_d = output_d.to(torch.float16).to(torch.float32)
+ output_dmin = output_dmin.to(torch.float16).to(torch.float32)
+ d_tmp = output_d * q_scales.to(torch.float32)
+ dm_tmp = output_dmin * q_mins.to(torch.float32)
+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))
all_L = (
- blocks.add_(mins.unsqueeze(-1))
- .mul_(get_reciprocal(scales.unsqueeze(-1)))
+ blocks.add_(dm_tmp.unsqueeze(-1))
+ .mul_(get_reciprocal(d_tmp.unsqueeze(-1)))
.round_()
.clamp_(0, 31)
.to(torch.uint8)
@@ -909,9 +926,14 @@ def q5_k_quant_block(
output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)
q_scales = (scales * get_reciprocal(output_d)).round_().clamp_(0, 63).to(torch.uint8)
q_mins = (mins * get_reciprocal(output_dmin)).round_().clamp_(0, 63).to(torch.uint8)
+ output_d = output_d.to(torch.float16).to(torch.float32)
+ output_dmin = output_dmin.to(torch.float16).to(torch.float32)
+ d_tmp = output_d * q_scales.to(torch.float32)
+ dm_tmp = output_dmin * q_mins.to(torch.float32)
+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))
all_L = (
- blocks.add_(mins.unsqueeze(-1))
- .mul_(get_reciprocal(scales.unsqueeze(-1)))
+ blocks.add_(dm_tmp.unsqueeze(-1))
+ .mul_(get_reciprocal(d_tmp.unsqueeze(-1)))
.round_()
.clamp_(0, 31)
.to(torch.uint8)
@@ -952,8 +974,10 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,
output_d = d_scale.reshape(-1, 1).to(torch.float32)
rd = get_reciprocal(output_d)
output_scale = scales.mul(rd).round_().clamp_(max=127).to(torch.int8)
- rs = get_reciprocal(scales).unsqueeze_(-1) # unsqueeze for broadcasting
- all_L = blocks.mul_(rs).add_(32).round_().clamp_(0, 63).to(torch.uint8)
+ output_d = output_d.to(torch.float16).to(torch.float32)
+ d_tmp = output_d * output_scale.to(torch.float32)
+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))
+ all_L = blocks.mul_(get_reciprocal(d_tmp).unsqueeze_(-1)).add_(32).round_().clamp_(0, 63).to(torch.uint8)
elif original:
scales, all_L = make_qx_quants(blocks, bits=6, rmse_type=1, qw=None)
imax = abs(scales).argmax(dim=-1, keepdim=True)
@@ -978,7 +1002,10 @@ def q6_k_quant_block(blocks: np.array, scale=None, d_scale=None, original=False,
scales = scales.reshape((-1, QK_K // 16))
output_d = d_scale.reshape(-1, 1).to(torch.float32)
output_scale = (scales * get_reciprocal(output_d)).round_().clamp_(max=127).to(torch.int8)
- all_L = blocks.mul_(get_reciprocal(scales.unsqueeze(-1))).add_(32).round_().clamp_(0, 63).to(torch.uint8)
+ output_d = output_d.to(torch.float16).to(torch.float32)
+ d_tmp = output_d * output_scale.to(torch.float32)
+ d_tmp = torch.where(d_tmp != 0, d_tmp, torch.ones_like(d_tmp))
+ all_L = blocks.mul_(get_reciprocal(d_tmp).unsqueeze_(-1)).add_(32).round_().clamp_(0, 63).to(torch.uint8)
tmp_L = all_L.reshape(nb, 4, 64) & 0xF
output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)
diff --git a/auto_round/formats.py b/auto_round/formats.py
index 5dcd3ca6b..d16de4756 100644
--- a/auto_round/formats.py
+++ b/auto_round/formats.py
@@ -95,12 +95,23 @@ def _check_compatibility(formats: list[str], ar: BaseCompressor):
if gguf_format_name.lower().endswith("mixed"):
gguf_format_name = gguf_format_name.lower().replace("_mixed", "_s")
if any([f.lower() not in ["fake", gguf_format_name.lower()] for f in formats]):
- tmp_format_name = gguf_format_name.lower() if "fake" not in formats else f"{gguf_format_name.lower()},fake"
- logger.warning(
- f"reset format {','.join(formats)} to {tmp_format_name} "
- f"since scheme {gguf_format_name} can only be exported to format {gguf_format_name.lower()} or fake"
- )
- formats = tmp_format_name.split(",")
+ has_gguf_format = any(f.lower().startswith("gguf") for f in formats if f.lower() != "fake")
+ if has_gguf_format:
+ logger.warning(
+ f"scheme {gguf_format_name} is GGUF, but format {','.join(formats)} specifies "
+ f"a different GGUF type. The scheme-driven per-layer quantization may differ from the "
+ f"file-level GGUF format type."
+ )
+ else:
+ tmp_format_name = (
+ gguf_format_name.lower() if "fake" not in formats else (f"{gguf_format_name.lower()},fake")
+ )
+ logger.warning(
+ f"reset format {','.join(formats)} to {tmp_format_name} "
+ f"since scheme {gguf_format_name} can only be exported to format "
+ f"{gguf_format_name.lower()} or fake"
+ )
+ formats = tmp_format_name.split(",")
if isinstance(ar.group_size, tuple) and any(["auto_round" in f.lower() for f in formats]):
logger.warning(
@@ -149,6 +160,11 @@ def get_formats(
formats = [fmt for fmt in formats if fmt is not None]
+ # Ensure fake format is processed before GGUF — GGUF export may clear
+ # model weights via low_cpu_mem_usage, causing zeroed weights for fake.
+ if any(fmt.is_gguf() for fmt in formats) and any(fmt.is_fake() for fmt in formats):
+ formats.sort(key=lambda f: 0 if f.is_fake() else 1)
+
if len(formats) == 1 and formats[0].is_gguf() and ar.scale_dtype != torch.float32:
ar.scale_dtype = torch.float32
logger.info("change `scale_dtype` to `torch.float32` for gguf format")
@@ -795,7 +811,7 @@ class GGUFFormat(OutputFormat):
gguf_format, ar.layer_config, ar.model, quant_nontext_module=ar.quant_nontext_module
)
gguf_format = gguf_format.lower().replace("_mixed", "_s")
- if isinstance(scheme, str) and scheme.lower() != gguf_format:
+ if isinstance(scheme, str) and scheme.lower() != gguf_format and not getattr(ar, "is_auto_scheme", False):
logger.warning(f"reset scheme {scheme.lower()} to {gguf_format} for gguf format export")
ar.scheme = gguf_format
self.output_format = gguf_format