PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf format gemma4 High RAM and glm-4.7 error (#2084)
Heng Guo0212c163650f
diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.pyindex d050e3861..c355cb566 100644--- a/auto_round/compressors/orchestrator.py+++ b/auto_round/compressors/orchestrator.py@@ -433,7 +433,6 @@ class CompressionOrchestrator(BaseOrchestrator):memory_monitor.log_summary()pbar.update(1)- cnt = 1remain_layer_names = []block_name_set = set(name for block in all_blocks for name in block)for n, m in self.model.named_modules():@@ -446,10 +445,10 @@ class CompressionOrchestrator(BaseOrchestrator):for name in remain_layer_names:logger.info(f"Quantizing remaining layer {name} on CPU.")self.alg_composer.compress_layer_outside_block(get_module(self.model, name))- cnt += 1- if cnt % 10 == 0:- clear_memory()- memory_monitor.log_summary()+ # Outside-block layers (embed_tokens/lm_head/etc.) are typically few so just+ # log a summary after each one.+ clear_memory()+ memory_monitor.log_summary()# Convert remaining fp8convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device)diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 060a9a49d..b68c05d02 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -18,7 +18,12 @@ import torchfrom auto_round.data_type.register import register_dtypefrom auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_stefrom auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K-from auto_round.export.export_to_gguf.packing import make_q3_quants, make_qx_quants, make_qx_quants_chunk+from auto_round.export.export_to_gguf.packing import (+ _LARGE_TENSOR_BYTES_THRESHOLD,+ make_q3_quants,+ make_qx_quants,+ make_qx_quants_chunk,+)from auto_round.logger import loggerfrom auto_round.utils import get_reciprocalfrom auto_round.utils.device import clear_memory@@ -381,12 +386,66 @@ def quant_tensor_asym_dq(return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}+def _imatrix_row_pattern(imatrix: torch.Tensor, group_shape) -> torch.Tensor:+ """Reshape a per-input-channel imatrix into the small, non-materialized "one row of+ groups" repeat unit, instead of broadcasting+reshaping it to the full weight tensor's+ shape.++ An imatrix is computed per input channel (column) and is identical across every output+ row, so once it's grouped the same way the weight tensor is (into blocks/groups of+ ``group_shape`` elements each), the resulting pattern of length ``blocks_per_row =+ imatrix.numel() // prod(group_shape)`` simply repeats for every output row. Any global+ block/group index ``i`` (0-based, row-major over the weight tensor) maps to+ ``pattern[i % blocks_per_row]`` — this holds for ANY quant type/group size, since GGUF+ requires ``in_features % prod(group_shape) == 0`` (blocks never straddle two rows).++ Building only this small ``(blocks_per_row, *group_shape)`` tensor - instead of+ ``imatrix.expand(...).reshape(weight.shape)``, which PyTorch cannot represent as a view+ once the broadcast dimension has to be reshaped/merged and so silently materializes a+ full copy as large as the weight tensor - keeps the up-front cost independent of+ out_features, no matter how large a single tensor (e.g. an oversized vocab embedding) is.+ """+ if isinstance(group_shape, int):+ group_shape = (group_shape,)+ group_numel = 1+ for d in group_shape:+ group_numel *= d+ blocks_per_row = imatrix.numel() // group_numel+ return imatrix.reshape(blocks_per_row, *group_shape)+++def _gather_row_pattern(row_pattern: torch.Tensor, start: int, end: int) -> torch.Tensor:+ """Materialize only the ``[start:end)`` slice of a full, row-repeating broadcast, by+ gathering from the small ``row_pattern`` (see ``_imatrix_row_pattern``) with the block+ index taken modulo ``blocks_per_row``. Cheap regardless of how large ``end - start`` is+ relative to the full (un-chunked) tensor, since ``row_pattern`` itself never grows.+ """+ bpr = row_pattern.shape[0]+ idx = torch.arange(start, end, device=row_pattern.device) % bpr+ return row_pattern.index_select(0, idx)++def _imatrix_handle_zero(- imatrix: Union[torch.Tensor, float], weight: torch.Tensor, bits: int, group_size: Union[int, None] = None+ imatrix: Union[torch.Tensor, float],+ weight: torch.Tensor,+ bits: int,+ group_size: Union[int, None] = None,+ raw_imatrix: torch.Tensor = None,):if not isinstance(imatrix, torch.Tensor):return imatrix+ # `imatrix` here is normally a broadcast-expanded VIEW of the much smaller per-block+ # `raw_imatrix` (no extra memory), built via `.expand(...).reshape(weight.shape)`. But the+ # `.reshape(-1, imatrix.shape[-1])` a few lines below merges the broadcast dimension into a+ # real one, which forces PyTorch to fully materialize a tensor as large as `weight` itself+ # (e.g. an extra ~11GB for an ~11GB embedding tensor). The common case - calibration data+ # with no zero-valued activations - never needs any of that; check for zeros on the small,+ # pre-broadcast `raw_imatrix` first and skip the expensive materialization entirely then.+ probe = raw_imatrix if raw_imatrix is not None else imatrix+ if torch.min(probe) != 0:+ return imatrix+if group_size is None:group_size = 16 if bits == 2 else 32imatrix = imatrix.reshape(-1, imatrix.shape[-1]).clone()@@ -464,10 +523,24 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatri5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},}- weights = imatrix.reshape(1, -1)- weights = weights.expand(tensor.numel() // weights.numel(), -1)- quant_weights = weights.reshape(tensor.shape)- quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits)+ # An imatrix is per input channel and identical across every output row, so+ # broadcasting it to `tensor.shape` (`.expand(...).reshape(...)`) forces PyTorch to+ # materialize a full copy as large as `tensor` itself - regardless of `split_num` -+ # since that materialization happens here, before any chunked search even starts.+ # For a single very large tensor (e.g. an oversized vocab embedding), that alone can+ # cost tens of extra GB. Zero-valued calibration activations are rare (and already+ # warned about elsewhere), so only fall back to the full materialization + explicit+ # zero-handling in that rare case; otherwise keep imatrix as the small, row-repeating+ # pattern and let the chunked search gather from it per chunk on demand.+ row_pattern = _imatrix_row_pattern(imatrix, tensor.shape[-1])+ quant_weights = None+ weights_row_pattern = row_pattern+ if torch.min(imatrix) == 0:+ weights = imatrix.reshape(1, -1)+ weights = weights.expand(tensor.numel() // weights.numel(), -1)+ quant_weights = weights.reshape(tensor.shape)+ quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits, raw_imatrix=imatrix)+ weights_row_pattern = None# sigma2 = torch.sum(torch.pow(tensor, 2), dim=-1, keepdim=True) / QK_K# if imatrix is None:@@ -487,6 +560,8 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrinstep=params["nstep"],use_mad=params["use_mad"],weights=quant_weights,+ weights_row_pattern=weights_row_pattern,+ split_num=split_num,v=v,)scale = scale.to(scale_dtype)@@ -494,7 +569,15 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrinmax = int(2.0**super_bits) - 1scale = scale.reshape(-1, super_group_size)wmin = wmin_0.reshape(-1, super_group_size)- sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)+ if quant_weights is not None:+ sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)+ else:+ # Same row-repeating trick for the (much smaller, one-value-per-block) sum: the+ # per-block sum of an imatrix broadcast repeats with the same `blocks_per_row`+ # period, so gather it from a tiny precomputed pattern instead of reducing a+ # full-size `quant_weights`.+ sum_pattern = row_pattern.sum(-1, keepdim=True)+ sum_quant_weights = _gather_row_pattern(sum_pattern, 0, tensor.shape[0]).reshape(-1, super_group_size)d_scale, q_scale = make_qp_quants(nmax, scale, sum_quant_weights)d_wmin, q_wmin = make_qp_quants(nmax, wmin, sum_quant_weights)@@ -543,7 +626,16 @@ def quant_tensor_gguf_asym_dq(orig_dtype = tensor.dtypemaxq = int(2.0**bits) - 1group_size = 16 if bits == 2 else 32- split_num = 1+ # The RTN-tuning path previously always ran the scale search unchunked (`split_num=1`),+ # unlike the export/packing path (`ggml_quant` in packing.py) which already chunks large+ # tensors. For very large single tensors (e.g. huge vocab embed/output projections),+ # `search_gguf_scale_min_asym`'s unchunked candidate search allocates several full-size+ # float32 buffers at once and can need tens of GB, causing OOM even though the model+ # itself is small. Chunk it here too once the tensor is large enough to matter - but only+ # then: chunking has real per-chunk Python-loop/kernel-launch overhead, so applying it to+ # every ordinary attention/FFN weight (which never risked OOM in the first place) would+ # just slow those down for no benefit. See `_LARGE_TENSOR_BYTES_THRESHOLD`.+ split_num = 16 if tensor.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD else 1tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)@@ -581,7 +673,16 @@ def quant_tensor_gguf_asym_dq(# TODO consolidate iterative_wls_quant_search_chunk and non-chunkdef iterative_wls_quant_search_chunk(- data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, split_num=1, v=0+ data,+ bits=4,+ rrmin=-1.0,+ rdelta=0.1,+ nstep=20,+ use_mad=False,+ weights=None,+ split_num=1,+ v=0,+ weights_row_pattern=None,):dtype = torch.float32data = data.to(dtype)@@ -599,7 +700,12 @@ def iterative_wls_quant_search_chunk(for start in range(0, data.shape[0], chunk_size):end = min(start + chunk_size, data.shape[0])chunk = data[start:end]- chunk_weights = weights if isinstance(weights, float) else weights[start:end]+ if weights_row_pattern is not None:+ # See `_imatrix_row_pattern`/`_gather_row_pattern`: gather this chunk's weights+ # from the small repeating pattern instead of slicing a full-size `weights`.+ chunk_weights = _gather_row_pattern(weights_row_pattern, start, end).to(dtype)+ else:+ chunk_weights = weights if isinstance(weights, float) else weights[start:end]v_chunk = v[start:end] if v_is_tensor else v# Pre-allocate reusable buffers to avoid new allocations@@ -701,7 +807,16 @@ def iterative_wls_quant_search_chunk(def iterative_wls_quant_search(- data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, split_num=1, v=0+ data,+ bits=4,+ rrmin=-1.0,+ rdelta=0.1,+ nstep=20,+ use_mad=False,+ weights=None,+ split_num=1,+ v=0,+ weights_row_pattern=None,):"""Adapted from Llamacpp. Performs iterative weighted least squares quantization search.@@ -715,6 +830,8 @@ def iterative_wls_quant_search(weights (torch.Tensor): Weight matrix for each element.v: Optional rounding perturbation (scalar or tensor with the sameshape as ``data``) for AutoRound-style tuning.+ weights_row_pattern: Optional small, row-repeating weight pattern (see+ ``_imatrix_row_pattern``) used instead of a full-size ``weights`` tensor.Returns:Tuple: (Optimal scale tensor, optimal minimum value tensor)@@ -732,6 +849,7 @@ def iterative_wls_quant_search(weights=weights,split_num=split_num,v=v,+ weights_row_pattern=weights_row_pattern,)@@ -747,13 +865,26 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num, v=0scale, int_w = make_qx_quants_chunk(tensor, bits=bits, rmse_type=1, qw=None, split_num=split_num, v=v)else:imatrix = imatrix.to(tensor.device)- weights = imatrix.reshape(1, -1)- weights = weights.expand(tensor.numel() // weights.numel(), -1)- quant_weights = weights.reshape(tensor.shape)-- quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits)-- scale, int_w = make_qx_quants_chunk(tensor, bits=bits, rmse_type=1, qw=quant_weights, split_num=split_num, v=v)+ # See the analogous comment in `search_gguf_scale_min_asym`: broadcasting+reshaping+ # imatrix to `tensor.shape` forces a full materialization proportional to the whole+ # tensor, independent of `split_num`, since it happens before any chunking starts.+ # Only fall back to that full materialization (needed for `_imatrix_handle_zero`'s+ # explicit zero-row replacement) in the rare case calibration actually produced+ # zero-valued entries; otherwise pass the small row-repeating pattern straight into+ # the already-chunked `make_qx_quants_chunk`.+ row_pattern = _imatrix_row_pattern(imatrix, tuple(tensor.shape[1:]))+ quant_weights = None+ qw_row_pattern = row_pattern+ if torch.min(imatrix) == 0:+ weights = imatrix.reshape(1, -1)+ weights = weights.expand(tensor.numel() // weights.numel(), -1)+ quant_weights = weights.reshape(tensor.shape)+ quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits, raw_imatrix=imatrix)+ qw_row_pattern = None++ scale, int_w = make_qx_quants_chunk(+ tensor, bits=bits, rmse_type=1, qw=quant_weights, qw_row_pattern=qw_row_pattern, split_num=split_num, v=v+ )if split_num > 1:clear_memory(device_list=[tensor.device])return scale@@ -805,6 +936,16 @@ def quant_tensor_gguf_sym_dq(block_size, type_size = GGML_QUANT_SIZES[ggml_type]tensor = tensor.to(torch.float32)n_blocks = tensor.nelement() // block_size+ # Chunk the candidate-scale search for very large single tensors (e.g. huge vocab+ # embed/output projections). This search allocates several full-size float32 buffers+ # at once; unchunked, that can need tens of GB for a single tensor and OOM even though+ # the overall model is small. Callers pass split_num=1 by default; bump it here once the+ # tensor is large enough to matter (see `_LARGE_TENSOR_BYTES_THRESHOLD`), mirroring the+ # heuristic already used in the export/packing path (`ggml_quant` in packing.py). Chunking+ # has real per-chunk overhead, so it's deliberately NOT applied below that threshold -+ # ordinary attention/FFN weights never needed it and would just get slower for no benefit.+ if split_num == 1 and tensor.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD:+ split_num = 16# (nb, 16, 16)tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)if iter is None:diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex a2a1bfa97..93c1b7e63 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -202,6 +202,17 @@ def get_restored_tensors(cls) -> Iterator[RestoredTensor]:yield RestoredTensor(name, lambda tensor=tensor: tensor, pending.hf_names, pending.transform_kind)+# gguf's canonical (architecture-independent) tensor names for the three merged MoE FFN+# weights -> the HF checkpoint weight-kind token used inside each per-expert weight's name+# (see e.g. glm.py/deepseek.py/qwen.py `modify_tensors`, which always buffer/merge experts as+# "down_proj", "gate_proj", "up_proj").+_MOE_EXP_SUFFIX_TO_HF_WNAME = {+ "ffn_gate_exps.weight": "gate_proj",+ "ffn_up_exps.weight": "up_proj",+ "ffn_down_exps.weight": "down_proj",+}++def _quant_data_with_args(data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None, device=None):@@ -1000,43 +1011,54 @@ def prepare_tensors(cls):data_qtype = gguf.GGMLQuantizationType.F16data = gguf.quants.quantize(data, data_qtype)else:- # for deepseek v2+ # for deepseek v2 / GLM_DSA style MLA models: kv_b_proj is split (by row) into+ # k_b_proj and v_b_proj, and k_b_proj is additionally transposed so its rows+ # become the lora-rank axis. The stored scale/zp/wmin/d_scale/d_wmin are grouped+ # along the (pre-transpose) lora-rank axis; after the k_b_proj transpose, that+ # group axis lands on the row axis, where a single GGUF quant block cannot share+ # one scale value across multiple output rows. Reusing those tensors as-is (as the+ # code used to do, via a plain view/split/transpose mirroring the weight transform)+ # produces a scale whose element count no longer matches the number of quant blocks+ # (e.g. a "size of tensor a must match tensor b" crash in q8_0/q4_k packing) or,+ # worse, a wrong-but-same-sized scale. So always let ggml_quant recompute scale/zp+ # for both k_b_proj and v_b_proj from the split (and possibly transposed) data.if hf_name.endswith("kv_b_proj.weight") and cls.model_arch.name in ("DEEPSEEK2", "GLM_DSA"):layer_name = hf_name[: -len(".weight")]module = get_module(cls.model, layer_name)- n_head_kv = cls.hparams["num_key_value_heads"]- v_head_dim = cls.hparams["v_head_dim"]- qk_nope_head_dim = cls.hparams["qk_nope_head_dim"]++ name_kb = modify_name.replace("kv_b_proj", "k_b_proj")+ is_k_b = new_name == cls.map_tensor_name(name_kb)attr_list = {"scale": None, "zp": None, "d_scale": None, "wmin": None, "d_wmin": None}- for attr in attr_list:- if hasattr(module, attr) or hasattr(module, "w_" + attr):- if attr in ["scale", "zp"]:- attr_tensor = getattr(module, attr)- else:- attr_tensor = getattr(module, "w_" + attr)- if attr_tensor is None or not isinstance(attr_tensor, torch.Tensor):- continue- kv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)- k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)- k_b = k_b.transpose(1, 2)-- name_kb = modify_name.replace("kv_b_proj", "k_b_proj")- if new_name == cls.map_tensor_name(name_kb):- attr_list[attr] = k_b- else:- attr_list[attr] = v_b- attr_list["imatrix"] = module.imatrix if hasattr(module, "imatrix") else None+ # v_b_proj is only row-subset, not transposed, so its column axis (lora-rank)+ # is unchanged and the per-input-channel imatrix still lines up; k_b_proj's+ # column axis becomes qk_nope_head_dim after the transpose, so imatrix (indexed+ # by lora-rank) would no longer apply to the right axis and must be dropped too.+ imatrix = getattr(module, "imatrix", None)+ attr_list["imatrix"] = imatrix if (not is_k_b and isinstance(imatrix, torch.Tensor)) else Nonedata = _quant_data_with_args(data_torch, data_qtype, device=device, **attr_list)# for MOE modelelif len(data_torch.shape) == 3 and len(re.findall(r"\d+", hf_name)) == 2:+ # `hf_name` is the checkpoint name of whichever single expert tensor happened+ # to complete the buffering in modify_tensors (it collects down_proj/gate_proj/+ # up_proj for all experts before yielding); all three merged tensors+ # (ffn_down_exps/ffn_gate_exps/ffn_up_exps) are yielded from that SAME call, so+ # `hf_name`'s own weight-kind token (e.g. "up_proj") only matches the tensor kind+ # actually being packed here by coincidence. Reusing it as-is for the other two+ # kinds silently fetches a DIFFERENT expert weight's module, whose scale/zp/+ # imatrix are then wrongly applied here (same shape, unrelated values) instead of+ # safely failing, so this must be corrected before looking up per-expert modules.+ hf_w_name = next((v for k, v in _MOE_EXP_SUFFIX_TO_HF_WNAME.items() if new_name.endswith(k)), None)new_data = []for idx, arr in enumerate(data_torch):arr_name = hf_name.split(".")for i in range(len(arr_name) - 1, -1, -1):if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):arr_name[i] = str(idx)+ if hf_w_name is not None and i + 1 < len(arr_name):+ arr_name[i + 1] = hf_w_name+ breakarr_name = ".".join(arr_name)arr, data_qtype = _quant_data(cls, arr, data_qtype, arr_name, modify_name, new_name, bid, device=devicediff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex fa0210868..4186aa4a2 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -20,6 +20,16 @@ from auto_round.utils import clear_memory, get_reciprocalGGML_QUANT_TYPE = {}+# Only chunk (via `split_num`) tensors whose fp32 footprint exceeds this many bytes. Chunking+# trades some real per-chunk Python-loop/GPU-kernel-launch overhead for bounded peak memory, so+# it should only kick in for genuinely oversized single tensors (e.g. an unusually large vocab+# embedding, which can be several GB) - not every ordinary attention/FFN weight, which would pay+# that overhead for no memory benefit (they're already only tens to a few hundred MB, even for+# very large dense models). 1GB comfortably separates the two: it's well above the largest+# ordinary per-tensor weight sizes seen in current dense/MoE LLMs, while still catching the+# outsized embeddings that actually risk OOM.+_LARGE_TENSOR_BYTES_THRESHOLD = 1_000_000_000+def register_qtype(name):@@ -75,7 +85,12 @@ def ggml_quant(data = data.to(torch.float32).to(device)shape = data.shapen_blocks = data.nelement() // block_size- split_num = 16 if max(data.shape) > 100_000 else 1+ # Only chunk genuinely oversized tensors (e.g. an unusually large vocab embedding, multiple+ # GB in fp32): chunking has real per-chunk Python-loop/kernel-launch overhead, so a low+ # threshold (e.g. based on block count alone) would slow down completely ordinary,+ # comfortably-sized attention/FFN weights for no memory benefit. See+ # `_LARGE_TENSOR_BYTES_THRESHOLD` above (also reused by `data_type/gguf.py`).+ split_num = 16 if data.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD else 1blocks = data.reshape((n_blocks, block_size))def _safe_reshape(t):@@ -132,7 +147,7 @@ def torch_roundf(n):return torch.sign(n) * b-def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):+def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0, qw_row_pattern=None):"""Extreme VRAM-optimized version of quantization.@@ -144,6 +159,12 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):v: Optional rounding perturbation. Either a scalar or a tensor with thesame shape as ``data``; added before each ``round`` to supportAutoRound-style tuning.+ qw_row_pattern: Optional small "one row of blocks" importance-weight tensor+ (shape ``(blocks_per_row, *group_shape)``), used instead of a full-size+ ``qw`` when the weighting is an imatrix broadcast that repeats every+ ``blocks_per_row`` blocks along dim 0 (see callers in ``data_type/gguf.py``).+ Avoids ever materializing a tensor as large as ``data`` just for the+ (constant, repeating) importance weights."""nmax = 2 ** (bits - 1)scales_list = []@@ -151,6 +172,7 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):chunk_size = (data.shape[0] + split_num - 1) // split_num# A 0-dim tensor (scalar tensor) cannot be sliced; treat it like a Python scalar.v_is_tensor = isinstance(v, torch.Tensor) and v.dim() > 0+ bpr = qw_row_pattern.shape[0] if qw_row_pattern is not None else Nonefor start in range(0, data.shape[0], chunk_size):end = min(start + chunk_size, data.shape[0])chunk = data[start:end] # Slice a batch chunk to reduce memory footprint@@ -183,7 +205,13 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):# Compute weighting tensor w based on rmse_typeif qw is not None:- w = qw+ w = qw[start:end]+ elif qw_row_pattern is not None:+ # The importance weights repeat every `bpr` blocks (broadcast across output rows);+ # gather this chunk's slice from the small per-row-of-blocks pattern instead of a+ # full-size `qw` tensor.+ row_idx = torch.arange(start, end, device=data.device) % bpr+ w = qw_row_pattern.index_select(0, row_idx)elif rmse_type == 1:w = chunk * chunkelif rmse_type == 2:diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.pyindex 192a68443..d43a0962e 100644--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py@@ -52,9 +52,9 @@ from auto_round.wrapper import WrapperWALayerfrom .config import check_compressed_tensors_supportedfrom .export_to_static_fp import (- _append_attention_group,_configure_gaudi2_fp8_dtype,_construct_kv_scheme,+ _get_attention_config,_use_fp8_attention,_use_fp8_kv,)@@ -204,7 +204,9 @@ def _build_mixed_fp_quantization_config(use_fp8_attention = _use_fp8_attention(static_attention_dtype)if use_fp8_attention:- _append_attention_group(config_groups, model)+ attention_config = _get_attention_config(model)+ else:+ attention_config = Nonequantization_config = initialize_quantization(scheme=None,config_groups=config_groups,@@ -219,6 +221,8 @@ def _build_mixed_fp_quantization_config(quantization_config["config_groups"][group_name]["format"] = fmtquantization_config["provider"] = "auto-round"_configure_gaudi2_fp8_dtype(quantization_config)+ if attention_config is not None:+ quantization_config["attention_input_activations"] = attention_configreturn quantization_config@@ -347,11 +351,15 @@ def save_quantized_as_fp(ignore=ignore,)if use_fp8_attention:- _append_attention_group(quantization_config.config_groups, model)+ attention_config = _get_attention_config(model)+ else:+ attention_config = Nonesetattr(quantization_config, "format", format)quantization_config = quantization_config.to_dict()quantization_config["provider"] = "auto-round"_configure_gaudi2_fp8_dtype(quantization_config)+ if attention_config is not None:+ quantization_config["attention_input_activations"] = attention_configif hasattr(model, "config"):model.config.quantization_config = quantization_configif output_dir is None:diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.pyindex 6bb055d57..82cf6d2a2 100644--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py@@ -119,20 +119,23 @@ def _get_attention_targets(model: torch.nn.Module) -> list[str]:return attention_targets-def _append_attention_group(config_groups: dict, model: torch.nn.Module) -> None:- from compressed_tensors.quantization import QuantizationScheme # pylint: disable=E0401+def _get_attention_config(model: torch.nn.Module) -> dict | None:+ """Return attention FP8 config as a standalone dict for the top-level+ ``attention_input_activations`` field.+ Keeping this out of ``config_groups`` prevents compressed_tensors from+ trying to compress attention modules (which have no weight tensor to+ compress) during model loading.+ """attention_targets = _get_attention_targets(model)if len(attention_targets) == 0:- logger.warning("Skipping FP8 attention config group because no attention modules were detected.")- return+ logger.warning("Skipping FP8 attention config because no attention modules were detected.")+ return None- group_name = f"group_{len(config_groups)}"- config_groups[group_name] = QuantizationScheme(- targets=attention_targets,- input_activations=_construct_fp8_args(),- weights=None,- )+ return {+ "targets": attention_targets,+ "input_activations": _construct_fp8_args().model_dump(),+ }def _configure_gaudi2_fp8_dtype(quantization_config: dict) -> None:@@ -234,16 +237,16 @@ def save_quantized_as_static_fp(config_groups = {}scheme = QuantizationScheme(targets=targets, **scheme_args)config_groups["group_0"] = scheme- if _use_fp8_attention(serialization_dict.get("static_attention_dtype", None)):- _append_attention_group(config_groups, model)+ use_fp8_attention = _use_fp8_attention(serialization_dict.get("static_attention_dtype", None))+ if use_fp8_attention:+ attention_config = _get_attention_config(model)+ else:+ attention_config = Nonequantization_config = QuantizationConfig(config_groups=config_groups,kv_cache_scheme=(_construct_kv_scheme()- if (- _use_fp8_kv(serialization_dict.get("static_kv_dtype", None))- or _use_fp8_attention(serialization_dict.get("static_attention_dtype", None))- )+ if (_use_fp8_kv(serialization_dict.get("static_kv_dtype", None)) or use_fp8_attention)else None),quantization_status=QuantizationStatus.COMPRESSED,@@ -255,6 +258,8 @@ def save_quantized_as_static_fp(quantization_config["format"] = "float-quantized"if group_size > 0:quantization_config["config_groups"]["group_0"]["weights"]["group_size"] = group_size+ if attention_config is not None:+ quantization_config["attention_input_activations"] = attention_configif hasattr(model, "config"):model.config.quantization_config = quantization_configdiff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.pyindex e2a80b5a3..7982bf8bd 100644--- a/test/test_cpu/export/test_llmc_format.py+++ b/test/test_cpu/export/test_llmc_format.py@@ -162,42 +162,25 @@ class TestLLMC:with open(os.path.join(quantized_model_path, "config.json")) as f:saved_config = json.load(f)- saved_groups = saved_config["quantization_config"]["config_groups"]- attention_group = None- for group in saved_groups.values():- if "Linear" not in group["targets"]:- attention_group = group- break-- assert attention_group is not None- assert attention_group["targets"] == [compressed_model.model.decoder.layers[0].self_attn.__class__.__name__]- assert attention_group["weights"] is None- assert attention_group["input_activations"]["num_bits"] == 8- assert attention_group["input_activations"]["type"] == "float"- assert attention_group["input_activations"]["strategy"] == "tensor"- assert attention_group["input_activations"]["dynamic"] is False- assert attention_group["input_activations"]["symmetric"] is True+ # Attention config is stored in a dedicated top-level field (not config_groups).+ attention_config = saved_config["quantization_config"]["attention_input_activations"]+ assert attention_config is not None+ assert attention_config["targets"] == [compressed_model.model.decoder.layers[0].self_attn.__class__.__name__]+ assert attention_config["input_activations"]["num_bits"] == 8+ assert attention_config["input_activations"]["type"] == "float"+ assert attention_config["input_activations"]["strategy"] == "tensor"+ assert attention_config["input_activations"]["dynamic"] is False+ assert attention_config["input_activations"]["symmetric"] is Trueassert saved_config["quantization_config"]["kv_cache_scheme"] is not Nonequantization_config = transformers.AutoConfig.from_pretrained(quantized_model_path, trust_remote_code=True).quantization_configconfig_groups = quantization_config["config_groups"]- attention_group = None- for group in config_groups.values():- targets = group["targets"]- if "Linear" not in targets:- attention_group = group- break-- assert attention_group is not None- assert attention_group["targets"] == [compressed_model.model.decoder.layers[0].self_attn.__class__.__name__]- assert attention_group["weights"] is None- assert attention_group["input_activations"]["num_bits"] == 8- assert attention_group["input_activations"]["type"] == "float"- assert attention_group["input_activations"]["strategy"] == "tensor"- assert attention_group["input_activations"]["dynamic"] is False- assert attention_group["input_activations"]["symmetric"] is True+ # Only Linear layers should be in config_groups.+ for group_targets in (g["targets"] for g in config_groups.values()):+ assert "Linear" in group_targets, f"Unexpected non-Linear targets in config_groups: {group_targets}"+assert quantization_config["kv_cache_scheme"] is not Noneassert getattr(compressed_model.model.decoder.layers[0].self_attn, "q_scale", None) is not None@@ -293,46 +276,51 @@ def test_llmcompressor_static_fp8_attention_config(dataloader, tmp_path):with open(os.path.join(quantized_model_path, "config.json")) as f:saved_config = json.load(f)- saved_groups = saved_config["quantization_config"]["config_groups"]- attention_group = None- for group in saved_groups.values():- if "Linear" not in group["targets"]:- attention_group = group- break-- assert attention_group is not None- assert attention_group["weights"] is None- assert attention_group["input_activations"]["num_bits"] == 8- assert attention_group["input_activations"]["type"] == "float"- assert attention_group["input_activations"]["strategy"] == "tensor"- assert attention_group["input_activations"]["dynamic"] is False- assert attention_group["input_activations"]["symmetric"] is True++ # Attention config is stored in a dedicated top-level field (not config_groups)+ # to prevent compressed_tensors from trying to compress attention modules at load time.+ attention_config = saved_config["quantization_config"]["attention_input_activations"]+ assert attention_config is not None+ assert "Linear" not in attention_config["targets"]+ assert attention_config["input_activations"]["num_bits"] == 8+ assert attention_config["input_activations"]["type"] == "float"+ assert attention_config["input_activations"]["strategy"] == "tensor"+ assert attention_config["input_activations"]["dynamic"] is False+ assert attention_config["input_activations"]["symmetric"] is Trueassert saved_config["quantization_config"]["kv_cache_scheme"] is not None+ # Verify the model can be loaded without crashing.model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", trust_remote_code=True)quantization_config = model.config.quantization_configconfig = quantization_config.to_dict() if hasattr(quantization_config, "to_dict") else quantization_configconfig_groups = config["config_groups"]+ # Only Linear layers should be in config_groups; attention modules use the+ # separate attention_input_activations field.assert "group_0" in config_groups- attention_group = None- for group in config_groups.values():- targets = group["targets"]- if "Linear" not in targets:- attention_group = group- break-- assert attention_group is not None- assert attention_group["targets"] == [model.model.layers[0].self_attn.__class__.__name__]- assert attention_group["weights"] is None- assert attention_group["input_activations"]["num_bits"] == 8- assert attention_group["input_activations"]["type"] == "float"- assert attention_group["input_activations"]["strategy"] == "tensor"- assert attention_group["input_activations"]["dynamic"] is False- assert attention_group["input_activations"]["symmetric"] is True- assert getattr(model.model.layers[0].self_attn, "q_scale", None) is not None+ for group_targets in (g["targets"] for g in config_groups.values()):+ assert "Linear" in group_targets, f"Unexpected non-Linear targets in config_groups: {group_targets}"+assert config["kv_cache_scheme"] is not None+ # Verify KV cache quant parameters (q_scale, k_scale, v_scale) were saved+ # in the checkpoint. They show as UNEXPECTED on the loaded model because the+ # base HuggingFace architecture does not register them as standard parameters.+ import glob as _glob++ safetensors_files = _glob.glob(os.path.join(quantized_model_path, "*.safetensors"))+ assert len(safetensors_files) > 0, "No safetensors file found"+ from safetensors import safe_open as _safe_open++ has_q_scale = False+ for sf in safetensors_files:+ with _safe_open(sf, framework="pt") as f:+ for key in f.keys():+ if "self_attn.q_scale" in key:+ has_q_scale = True+ break+ assert has_q_scale, "q_scale not found in checkpoint"+def test_llmcompressor_mxfp8_export_packs_serially(tmp_path, monkeypatch):autoround = AutoRound(