Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gguf format gemma4 High RAM and glm-4.7 error (#2084)

Heng Guo0212c163650f
diff --git a/auto_round/compressors/orchestrator.py b/auto_round/compressors/orchestrator.py
index d050e3861..c355cb566 100644
--- a/auto_round/compressors/orchestrator.py
+++ b/auto_round/compressors/orchestrator.py
@@ -433,7 +433,6 @@ class CompressionOrchestrator(BaseOrchestrator):
memory_monitor.log_summary()
pbar.update(1)
- cnt = 1
remain_layer_names = []
block_name_set = set(name for block in all_blocks for name in block)
for n, m in self.model.named_modules():
@@ -446,10 +445,10 @@ class CompressionOrchestrator(BaseOrchestrator):
for name in remain_layer_names:
logger.info(f"Quantizing remaining layer {name} on CPU.")
self.alg_composer.compress_layer_outside_block(get_module(self.model, name))
- cnt += 1
- if cnt % 10 == 0:
- clear_memory()
- memory_monitor.log_summary()
+ # Outside-block layers (embed_tokens/lm_head/etc.) are typically few so just
+ # log a summary after each one.
+ clear_memory()
+ memory_monitor.log_summary()
# Convert remaining fp8
convert_module_to_hp_if_necessary(self.model, self.amp_dtype, self.device)
diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.py
index 060a9a49d..b68c05d02 100644
--- a/auto_round/data_type/gguf.py
+++ b/auto_round/data_type/gguf.py
@@ -18,7 +18,12 @@ import torch
from auto_round.data_type.register import register_dtype
from auto_round.data_type.utils import reshape_pad_tensor_by_group_size, revert_tensor_by_pad, round_ste
from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K
-from auto_round.export.export_to_gguf.packing import make_q3_quants, make_qx_quants, make_qx_quants_chunk
+from auto_round.export.export_to_gguf.packing import (
+ _LARGE_TENSOR_BYTES_THRESHOLD,
+ make_q3_quants,
+ make_qx_quants,
+ make_qx_quants_chunk,
+)
from auto_round.logger import logger
from auto_round.utils import get_reciprocal
from auto_round.utils.device import clear_memory
@@ -381,12 +386,66 @@ def quant_tensor_asym_dq(
return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}
+def _imatrix_row_pattern(imatrix: torch.Tensor, group_shape) -> torch.Tensor:
+ """Reshape a per-input-channel imatrix into the small, non-materialized "one row of
+ groups" repeat unit, instead of broadcasting+reshaping it to the full weight tensor's
+ shape.
+
+ An imatrix is computed per input channel (column) and is identical across every output
+ row, so once it's grouped the same way the weight tensor is (into blocks/groups of
+ ``group_shape`` elements each), the resulting pattern of length ``blocks_per_row =
+ imatrix.numel() // prod(group_shape)`` simply repeats for every output row. Any global
+ block/group index ``i`` (0-based, row-major over the weight tensor) maps to
+ ``pattern[i % blocks_per_row]`` — this holds for ANY quant type/group size, since GGUF
+ requires ``in_features % prod(group_shape) == 0`` (blocks never straddle two rows).
+
+ Building only this small ``(blocks_per_row, *group_shape)`` tensor - instead of
+ ``imatrix.expand(...).reshape(weight.shape)``, which PyTorch cannot represent as a view
+ once the broadcast dimension has to be reshaped/merged and so silently materializes a
+ full copy as large as the weight tensor - keeps the up-front cost independent of
+ out_features, no matter how large a single tensor (e.g. an oversized vocab embedding) is.
+ """
+ if isinstance(group_shape, int):
+ group_shape = (group_shape,)
+ group_numel = 1
+ for d in group_shape:
+ group_numel *= d
+ blocks_per_row = imatrix.numel() // group_numel
+ return imatrix.reshape(blocks_per_row, *group_shape)
+
+
+def _gather_row_pattern(row_pattern: torch.Tensor, start: int, end: int) -> torch.Tensor:
+ """Materialize only the ``[start:end)`` slice of a full, row-repeating broadcast, by
+ gathering from the small ``row_pattern`` (see ``_imatrix_row_pattern``) with the block
+ index taken modulo ``blocks_per_row``. Cheap regardless of how large ``end - start`` is
+ relative to the full (un-chunked) tensor, since ``row_pattern`` itself never grows.
+ """
+ bpr = row_pattern.shape[0]
+ idx = torch.arange(start, end, device=row_pattern.device) % bpr
+ return row_pattern.index_select(0, idx)
+
+
def _imatrix_handle_zero(
- imatrix: Union[torch.Tensor, float], weight: torch.Tensor, bits: int, group_size: Union[int, None] = None
+ imatrix: Union[torch.Tensor, float],
+ weight: torch.Tensor,
+ bits: int,
+ group_size: Union[int, None] = None,
+ raw_imatrix: torch.Tensor = None,
):
if not isinstance(imatrix, torch.Tensor):
return imatrix
+ # `imatrix` here is normally a broadcast-expanded VIEW of the much smaller per-block
+ # `raw_imatrix` (no extra memory), built via `.expand(...).reshape(weight.shape)`. But the
+ # `.reshape(-1, imatrix.shape[-1])` a few lines below merges the broadcast dimension into a
+ # real one, which forces PyTorch to fully materialize a tensor as large as `weight` itself
+ # (e.g. an extra ~11GB for an ~11GB embedding tensor). The common case - calibration data
+ # with no zero-valued activations - never needs any of that; check for zeros on the small,
+ # pre-broadcast `raw_imatrix` first and skip the expensive materialization entirely then.
+ probe = raw_imatrix if raw_imatrix is not None else imatrix
+ if torch.min(probe) != 0:
+ return imatrix
+
if group_size is None:
group_size = 16 if bits == 2 else 32
imatrix = imatrix.reshape(-1, imatrix.shape[-1]).clone()
@@ -464,10 +523,24 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatri
5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},
}
- weights = imatrix.reshape(1, -1)
- weights = weights.expand(tensor.numel() // weights.numel(), -1)
- quant_weights = weights.reshape(tensor.shape)
- quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits)
+ # An imatrix is per input channel and identical across every output row, so
+ # broadcasting it to `tensor.shape` (`.expand(...).reshape(...)`) forces PyTorch to
+ # materialize a full copy as large as `tensor` itself - regardless of `split_num` -
+ # since that materialization happens here, before any chunked search even starts.
+ # For a single very large tensor (e.g. an oversized vocab embedding), that alone can
+ # cost tens of extra GB. Zero-valued calibration activations are rare (and already
+ # warned about elsewhere), so only fall back to the full materialization + explicit
+ # zero-handling in that rare case; otherwise keep imatrix as the small, row-repeating
+ # pattern and let the chunked search gather from it per chunk on demand.
+ row_pattern = _imatrix_row_pattern(imatrix, tensor.shape[-1])
+ quant_weights = None
+ weights_row_pattern = row_pattern
+ if torch.min(imatrix) == 0:
+ weights = imatrix.reshape(1, -1)
+ weights = weights.expand(tensor.numel() // weights.numel(), -1)
+ quant_weights = weights.reshape(tensor.shape)
+ quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits, raw_imatrix=imatrix)
+ weights_row_pattern = None
# sigma2 = torch.sum(torch.pow(tensor, 2), dim=-1, keepdim=True) / QK_K
# if imatrix is None:
@@ -487,6 +560,8 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatri
nstep=params["nstep"],
use_mad=params["use_mad"],
weights=quant_weights,
+ weights_row_pattern=weights_row_pattern,
+ split_num=split_num,
v=v,
)
scale = scale.to(scale_dtype)
@@ -494,7 +569,15 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatri
nmax = int(2.0**super_bits) - 1
scale = scale.reshape(-1, super_group_size)
wmin = wmin_0.reshape(-1, super_group_size)
- sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)
+ if quant_weights is not None:
+ sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)
+ else:
+ # Same row-repeating trick for the (much smaller, one-value-per-block) sum: the
+ # per-block sum of an imatrix broadcast repeats with the same `blocks_per_row`
+ # period, so gather it from a tiny precomputed pattern instead of reducing a
+ # full-size `quant_weights`.
+ sum_pattern = row_pattern.sum(-1, keepdim=True)
+ sum_quant_weights = _gather_row_pattern(sum_pattern, 0, tensor.shape[0]).reshape(-1, super_group_size)
d_scale, q_scale = make_qp_quants(nmax, scale, sum_quant_weights)
d_wmin, q_wmin = make_qp_quants(nmax, wmin, sum_quant_weights)
@@ -543,7 +626,16 @@ def quant_tensor_gguf_asym_dq(
orig_dtype = tensor.dtype
maxq = int(2.0**bits) - 1
group_size = 16 if bits == 2 else 32
- split_num = 1
+ # The RTN-tuning path previously always ran the scale search unchunked (`split_num=1`),
+ # unlike the export/packing path (`ggml_quant` in packing.py) which already chunks large
+ # tensors. For very large single tensors (e.g. huge vocab embed/output projections),
+ # `search_gguf_scale_min_asym`'s unchunked candidate search allocates several full-size
+ # float32 buffers at once and can need tens of GB, causing OOM even though the model
+ # itself is small. Chunk it here too once the tensor is large enough to matter - but only
+ # then: chunking has real per-chunk Python-loop/kernel-launch overhead, so applying it to
+ # every ordinary attention/FFN weight (which never risked OOM in the first place) would
+ # just slow those down for no benefit. See `_LARGE_TENSOR_BYTES_THRESHOLD`.
+ split_num = 16 if tensor.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD else 1
tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)
@@ -581,7 +673,16 @@ def quant_tensor_gguf_asym_dq(
# TODO consolidate iterative_wls_quant_search_chunk and non-chunk
def iterative_wls_quant_search_chunk(
- data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, split_num=1, v=0
+ data,
+ bits=4,
+ rrmin=-1.0,
+ rdelta=0.1,
+ nstep=20,
+ use_mad=False,
+ weights=None,
+ split_num=1,
+ v=0,
+ weights_row_pattern=None,
):
dtype = torch.float32
data = data.to(dtype)
@@ -599,7 +700,12 @@ def iterative_wls_quant_search_chunk(
for start in range(0, data.shape[0], chunk_size):
end = min(start + chunk_size, data.shape[0])
chunk = data[start:end]
- chunk_weights = weights if isinstance(weights, float) else weights[start:end]
+ if weights_row_pattern is not None:
+ # See `_imatrix_row_pattern`/`_gather_row_pattern`: gather this chunk's weights
+ # from the small repeating pattern instead of slicing a full-size `weights`.
+ chunk_weights = _gather_row_pattern(weights_row_pattern, start, end).to(dtype)
+ else:
+ chunk_weights = weights if isinstance(weights, float) else weights[start:end]
v_chunk = v[start:end] if v_is_tensor else v
# Pre-allocate reusable buffers to avoid new allocations
@@ -701,7 +807,16 @@ def iterative_wls_quant_search_chunk(
def iterative_wls_quant_search(
- data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, split_num=1, v=0
+ data,
+ bits=4,
+ rrmin=-1.0,
+ rdelta=0.1,
+ nstep=20,
+ use_mad=False,
+ weights=None,
+ split_num=1,
+ v=0,
+ weights_row_pattern=None,
):
"""Adapted from Llamacpp. Performs iterative weighted least squares quantization search.
@@ -715,6 +830,8 @@ def iterative_wls_quant_search(
weights (torch.Tensor): Weight matrix for each element.
v: Optional rounding perturbation (scalar or tensor with the same
shape as ``data``) for AutoRound-style tuning.
+ weights_row_pattern: Optional small, row-repeating weight pattern (see
+ ``_imatrix_row_pattern``) used instead of a full-size ``weights`` tensor.
Returns:
Tuple: (Optimal scale tensor, optimal minimum value tensor)
@@ -732,6 +849,7 @@ def iterative_wls_quant_search(
weights=weights,
split_num=split_num,
v=v,
+ weights_row_pattern=weights_row_pattern,
)
@@ -747,13 +865,26 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num, v=0
scale, int_w = make_qx_quants_chunk(tensor, bits=bits, rmse_type=1, qw=None, split_num=split_num, v=v)
else:
imatrix = imatrix.to(tensor.device)
- weights = imatrix.reshape(1, -1)
- weights = weights.expand(tensor.numel() // weights.numel(), -1)
- quant_weights = weights.reshape(tensor.shape)
-
- quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits)
-
- scale, int_w = make_qx_quants_chunk(tensor, bits=bits, rmse_type=1, qw=quant_weights, split_num=split_num, v=v)
+ # See the analogous comment in `search_gguf_scale_min_asym`: broadcasting+reshaping
+ # imatrix to `tensor.shape` forces a full materialization proportional to the whole
+ # tensor, independent of `split_num`, since it happens before any chunking starts.
+ # Only fall back to that full materialization (needed for `_imatrix_handle_zero`'s
+ # explicit zero-row replacement) in the rare case calibration actually produced
+ # zero-valued entries; otherwise pass the small row-repeating pattern straight into
+ # the already-chunked `make_qx_quants_chunk`.
+ row_pattern = _imatrix_row_pattern(imatrix, tuple(tensor.shape[1:]))
+ quant_weights = None
+ qw_row_pattern = row_pattern
+ if torch.min(imatrix) == 0:
+ weights = imatrix.reshape(1, -1)
+ weights = weights.expand(tensor.numel() // weights.numel(), -1)
+ quant_weights = weights.reshape(tensor.shape)
+ quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits, raw_imatrix=imatrix)
+ qw_row_pattern = None
+
+ scale, int_w = make_qx_quants_chunk(
+ tensor, bits=bits, rmse_type=1, qw=quant_weights, qw_row_pattern=qw_row_pattern, split_num=split_num, v=v
+ )
if split_num > 1:
clear_memory(device_list=[tensor.device])
return scale
@@ -805,6 +936,16 @@ def quant_tensor_gguf_sym_dq(
block_size, type_size = GGML_QUANT_SIZES[ggml_type]
tensor = tensor.to(torch.float32)
n_blocks = tensor.nelement() // block_size
+ # Chunk the candidate-scale search for very large single tensors (e.g. huge vocab
+ # embed/output projections). This search allocates several full-size float32 buffers
+ # at once; unchunked, that can need tens of GB for a single tensor and OOM even though
+ # the overall model is small. Callers pass split_num=1 by default; bump it here once the
+ # tensor is large enough to matter (see `_LARGE_TENSOR_BYTES_THRESHOLD`), mirroring the
+ # heuristic already used in the export/packing path (`ggml_quant` in packing.py). Chunking
+ # has real per-chunk overhead, so it's deliberately NOT applied below that threshold -
+ # ordinary attention/FFN weights never needed it and would just get slower for no benefit.
+ if split_num == 1 and tensor.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD:
+ split_num = 16
# (nb, 16, 16)
tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)
if iter is None:
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index a2a1bfa97..93c1b7e63 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -202,6 +202,17 @@ def get_restored_tensors(cls) -> Iterator[RestoredTensor]:
yield RestoredTensor(name, lambda tensor=tensor: tensor, pending.hf_names, pending.transform_kind)
+# gguf's canonical (architecture-independent) tensor names for the three merged MoE FFN
+# weights -> the HF checkpoint weight-kind token used inside each per-expert weight's name
+# (see e.g. glm.py/deepseek.py/qwen.py `modify_tensors`, which always buffer/merge experts as
+# "down_proj", "gate_proj", "up_proj").
+_MOE_EXP_SUFFIX_TO_HF_WNAME = {
+ "ffn_gate_exps.weight": "gate_proj",
+ "ffn_up_exps.weight": "up_proj",
+ "ffn_down_exps.weight": "down_proj",
+}
+
+
def _quant_data_with_args(
data_torch, data_qtype, scale, zp, d_scale=None, wmin=None, d_wmin=None, imatrix=None, device=None
):
@@ -1000,43 +1011,54 @@ def prepare_tensors(cls):
data_qtype = gguf.GGMLQuantizationType.F16
data = gguf.quants.quantize(data, data_qtype)
else:
- # for deepseek v2
+ # for deepseek v2 / GLM_DSA style MLA models: kv_b_proj is split (by row) into
+ # k_b_proj and v_b_proj, and k_b_proj is additionally transposed so its rows
+ # become the lora-rank axis. The stored scale/zp/wmin/d_scale/d_wmin are grouped
+ # along the (pre-transpose) lora-rank axis; after the k_b_proj transpose, that
+ # group axis lands on the row axis, where a single GGUF quant block cannot share
+ # one scale value across multiple output rows. Reusing those tensors as-is (as the
+ # code used to do, via a plain view/split/transpose mirroring the weight transform)
+ # produces a scale whose element count no longer matches the number of quant blocks
+ # (e.g. a "size of tensor a must match tensor b" crash in q8_0/q4_k packing) or,
+ # worse, a wrong-but-same-sized scale. So always let ggml_quant recompute scale/zp
+ # for both k_b_proj and v_b_proj from the split (and possibly transposed) data.
if hf_name.endswith("kv_b_proj.weight") and cls.model_arch.name in ("DEEPSEEK2", "GLM_DSA"):
layer_name = hf_name[: -len(".weight")]
module = get_module(cls.model, layer_name)
- n_head_kv = cls.hparams["num_key_value_heads"]
- v_head_dim = cls.hparams["v_head_dim"]
- qk_nope_head_dim = cls.hparams["qk_nope_head_dim"]
+
+ name_kb = modify_name.replace("kv_b_proj", "k_b_proj")
+ is_k_b = new_name == cls.map_tensor_name(name_kb)
attr_list = {"scale": None, "zp": None, "d_scale": None, "wmin": None, "d_wmin": None}
- for attr in attr_list:
- if hasattr(module, attr) or hasattr(module, "w_" + attr):
- if attr in ["scale", "zp"]:
- attr_tensor = getattr(module, attr)
- else:
- attr_tensor = getattr(module, "w_" + attr)
- if attr_tensor is None or not isinstance(attr_tensor, torch.Tensor):
- continue
- kv_b = attr_tensor.view(n_head_kv, v_head_dim + qk_nope_head_dim, -1)
- k_b, v_b = torch.split(kv_b, [qk_nope_head_dim, v_head_dim], dim=1)
- k_b = k_b.transpose(1, 2)
-
- name_kb = modify_name.replace("kv_b_proj", "k_b_proj")
- if new_name == cls.map_tensor_name(name_kb):
- attr_list[attr] = k_b
- else:
- attr_list[attr] = v_b
- attr_list["imatrix"] = module.imatrix if hasattr(module, "imatrix") else None
+ # v_b_proj is only row-subset, not transposed, so its column axis (lora-rank)
+ # is unchanged and the per-input-channel imatrix still lines up; k_b_proj's
+ # column axis becomes qk_nope_head_dim after the transpose, so imatrix (indexed
+ # by lora-rank) would no longer apply to the right axis and must be dropped too.
+ imatrix = getattr(module, "imatrix", None)
+ attr_list["imatrix"] = imatrix if (not is_k_b and isinstance(imatrix, torch.Tensor)) else None
data = _quant_data_with_args(data_torch, data_qtype, device=device, **attr_list)
# for MOE model
elif len(data_torch.shape) == 3 and len(re.findall(r"\d+", hf_name)) == 2:
+ # `hf_name` is the checkpoint name of whichever single expert tensor happened
+ # to complete the buffering in modify_tensors (it collects down_proj/gate_proj/
+ # up_proj for all experts before yielding); all three merged tensors
+ # (ffn_down_exps/ffn_gate_exps/ffn_up_exps) are yielded from that SAME call, so
+ # `hf_name`'s own weight-kind token (e.g. "up_proj") only matches the tensor kind
+ # actually being packed here by coincidence. Reusing it as-is for the other two
+ # kinds silently fetches a DIFFERENT expert weight's module, whose scale/zp/
+ # imatrix are then wrongly applied here (same shape, unrelated values) instead of
+ # safely failing, so this must be corrected before looking up per-expert modules.
+ hf_w_name = next((v for k, v in _MOE_EXP_SUFFIX_TO_HF_WNAME.items() if new_name.endswith(k)), None)
new_data = []
for idx, arr in enumerate(data_torch):
arr_name = hf_name.split(".")
for i in range(len(arr_name) - 1, -1, -1):
if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):
arr_name[i] = str(idx)
+ if hf_w_name is not None and i + 1 < len(arr_name):
+ arr_name[i + 1] = hf_w_name
+ break
arr_name = ".".join(arr_name)
arr, data_qtype = _quant_data(
cls, arr, data_qtype, arr_name, modify_name, new_name, bid, device=device
diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.py
index fa0210868..4186aa4a2 100644
--- a/auto_round/export/export_to_gguf/packing.py
+++ b/auto_round/export/export_to_gguf/packing.py
@@ -20,6 +20,16 @@ from auto_round.utils import clear_memory, get_reciprocal
GGML_QUANT_TYPE = {}
+# Only chunk (via `split_num`) tensors whose fp32 footprint exceeds this many bytes. Chunking
+# trades some real per-chunk Python-loop/GPU-kernel-launch overhead for bounded peak memory, so
+# it should only kick in for genuinely oversized single tensors (e.g. an unusually large vocab
+# embedding, which can be several GB) - not every ordinary attention/FFN weight, which would pay
+# that overhead for no memory benefit (they're already only tens to a few hundred MB, even for
+# very large dense models). 1GB comfortably separates the two: it's well above the largest
+# ordinary per-tensor weight sizes seen in current dense/MoE LLMs, while still catching the
+# outsized embeddings that actually risk OOM.
+_LARGE_TENSOR_BYTES_THRESHOLD = 1_000_000_000
+
def register_qtype(name):
@@ -75,7 +85,12 @@ def ggml_quant(
data = data.to(torch.float32).to(device)
shape = data.shape
n_blocks = data.nelement() // block_size
- split_num = 16 if max(data.shape) > 100_000 else 1
+ # Only chunk genuinely oversized tensors (e.g. an unusually large vocab embedding, multiple
+ # GB in fp32): chunking has real per-chunk Python-loop/kernel-launch overhead, so a low
+ # threshold (e.g. based on block count alone) would slow down completely ordinary,
+ # comfortably-sized attention/FFN weights for no memory benefit. See
+ # `_LARGE_TENSOR_BYTES_THRESHOLD` above (also reused by `data_type/gguf.py`).
+ split_num = 16 if data.nelement() * 4 > _LARGE_TENSOR_BYTES_THRESHOLD else 1
blocks = data.reshape((n_blocks, block_size))
def _safe_reshape(t):
@@ -132,7 +147,7 @@ def torch_roundf(n):
return torch.sign(n) * b
-def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):
+def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0, qw_row_pattern=None):
"""
Extreme VRAM-optimized version of quantization.
@@ -144,6 +159,12 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):
v: Optional rounding perturbation. Either a scalar or a tensor with the
same shape as ``data``; added before each ``round`` to support
AutoRound-style tuning.
+ qw_row_pattern: Optional small "one row of blocks" importance-weight tensor
+ (shape ``(blocks_per_row, *group_shape)``), used instead of a full-size
+ ``qw`` when the weighting is an imatrix broadcast that repeats every
+ ``blocks_per_row`` blocks along dim 0 (see callers in ``data_type/gguf.py``).
+ Avoids ever materializing a tensor as large as ``data`` just for the
+ (constant, repeating) importance weights.
"""
nmax = 2 ** (bits - 1)
scales_list = []
@@ -151,6 +172,7 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):
chunk_size = (data.shape[0] + split_num - 1) // split_num
# A 0-dim tensor (scalar tensor) cannot be sliced; treat it like a Python scalar.
v_is_tensor = isinstance(v, torch.Tensor) and v.dim() > 0
+ bpr = qw_row_pattern.shape[0] if qw_row_pattern is not None else None
for start in range(0, data.shape[0], chunk_size):
end = min(start + chunk_size, data.shape[0])
chunk = data[start:end] # Slice a batch chunk to reduce memory footprint
@@ -183,7 +205,13 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):
# Compute weighting tensor w based on rmse_type
if qw is not None:
- w = qw
+ w = qw[start:end]
+ elif qw_row_pattern is not None:
+ # The importance weights repeat every `bpr` blocks (broadcast across output rows);
+ # gather this chunk's slice from the small per-row-of-blocks pattern instead of a
+ # full-size `qw` tensor.
+ row_idx = torch.arange(start, end, device=data.device) % bpr
+ w = qw_row_pattern.index_select(0, row_idx)
elif rmse_type == 1:
w = chunk * chunk
elif rmse_type == 2:
diff --git a/auto_round/export/export_to_llmcompressor/export_to_fp.py b/auto_round/export/export_to_llmcompressor/export_to_fp.py
index 192a68443..d43a0962e 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_fp.py
@@ -52,9 +52,9 @@ from auto_round.wrapper import WrapperWALayer
from .config import check_compressed_tensors_supported
from .export_to_static_fp import (
- _append_attention_group,
_configure_gaudi2_fp8_dtype,
_construct_kv_scheme,
+ _get_attention_config,
_use_fp8_attention,
_use_fp8_kv,
)
@@ -204,7 +204,9 @@ def _build_mixed_fp_quantization_config(
use_fp8_attention = _use_fp8_attention(static_attention_dtype)
if use_fp8_attention:
- _append_attention_group(config_groups, model)
+ attention_config = _get_attention_config(model)
+ else:
+ attention_config = None
quantization_config = initialize_quantization(
scheme=None,
config_groups=config_groups,
@@ -219,6 +221,8 @@ def _build_mixed_fp_quantization_config(
quantization_config["config_groups"][group_name]["format"] = fmt
quantization_config["provider"] = "auto-round"
_configure_gaudi2_fp8_dtype(quantization_config)
+ if attention_config is not None:
+ quantization_config["attention_input_activations"] = attention_config
return quantization_config
@@ -347,11 +351,15 @@ def save_quantized_as_fp(
ignore=ignore,
)
if use_fp8_attention:
- _append_attention_group(quantization_config.config_groups, model)
+ attention_config = _get_attention_config(model)
+ else:
+ attention_config = None
setattr(quantization_config, "format", format)
quantization_config = quantization_config.to_dict()
quantization_config["provider"] = "auto-round"
_configure_gaudi2_fp8_dtype(quantization_config)
+ if attention_config is not None:
+ quantization_config["attention_input_activations"] = attention_config
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
if output_dir is None:
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index 6bb055d57..82cf6d2a2 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -119,20 +119,23 @@ def _get_attention_targets(model: torch.nn.Module) -> list[str]:
return attention_targets
-def _append_attention_group(config_groups: dict, model: torch.nn.Module) -> None:
- from compressed_tensors.quantization import QuantizationScheme # pylint: disable=E0401
+def _get_attention_config(model: torch.nn.Module) -> dict | None:
+ """Return attention FP8 config as a standalone dict for the top-level
+ ``attention_input_activations`` field.
+ Keeping this out of ``config_groups`` prevents compressed_tensors from
+ trying to compress attention modules (which have no weight tensor to
+ compress) during model loading.
+ """
attention_targets = _get_attention_targets(model)
if len(attention_targets) == 0:
- logger.warning("Skipping FP8 attention config group because no attention modules were detected.")
- return
+ logger.warning("Skipping FP8 attention config because no attention modules were detected.")
+ return None
- group_name = f"group_{len(config_groups)}"
- config_groups[group_name] = QuantizationScheme(
- targets=attention_targets,
- input_activations=_construct_fp8_args(),
- weights=None,
- )
+ return {
+ "targets": attention_targets,
+ "input_activations": _construct_fp8_args().model_dump(),
+ }
def _configure_gaudi2_fp8_dtype(quantization_config: dict) -> None:
@@ -234,16 +237,16 @@ def save_quantized_as_static_fp(
config_groups = {}
scheme = QuantizationScheme(targets=targets, **scheme_args)
config_groups["group_0"] = scheme
- if _use_fp8_attention(serialization_dict.get("static_attention_dtype", None)):
- _append_attention_group(config_groups, model)
+ use_fp8_attention = _use_fp8_attention(serialization_dict.get("static_attention_dtype", None))
+ if use_fp8_attention:
+ attention_config = _get_attention_config(model)
+ else:
+ attention_config = None
quantization_config = QuantizationConfig(
config_groups=config_groups,
kv_cache_scheme=(
_construct_kv_scheme()
- if (
- _use_fp8_kv(serialization_dict.get("static_kv_dtype", None))
- or _use_fp8_attention(serialization_dict.get("static_attention_dtype", None))
- )
+ if (_use_fp8_kv(serialization_dict.get("static_kv_dtype", None)) or use_fp8_attention)
else None
),
quantization_status=QuantizationStatus.COMPRESSED,
@@ -255,6 +258,8 @@ def save_quantized_as_static_fp(
quantization_config["format"] = "float-quantized"
if group_size > 0:
quantization_config["config_groups"]["group_0"]["weights"]["group_size"] = group_size
+ if attention_config is not None:
+ quantization_config["attention_input_activations"] = attention_config
if hasattr(model, "config"):
model.config.quantization_config = quantization_config
diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.py
index e2a80b5a3..7982bf8bd 100644
--- a/test/test_cpu/export/test_llmc_format.py
+++ b/test/test_cpu/export/test_llmc_format.py
@@ -162,42 +162,25 @@ class TestLLMC:
with open(os.path.join(quantized_model_path, "config.json")) as f:
saved_config = json.load(f)
- saved_groups = saved_config["quantization_config"]["config_groups"]
- attention_group = None
- for group in saved_groups.values():
- if "Linear" not in group["targets"]:
- attention_group = group
- break
-
- assert attention_group is not None
- assert attention_group["targets"] == [compressed_model.model.decoder.layers[0].self_attn.__class__.__name__]
- assert attention_group["weights"] is None
- assert attention_group["input_activations"]["num_bits"] == 8
- assert attention_group["input_activations"]["type"] == "float"
- assert attention_group["input_activations"]["strategy"] == "tensor"
- assert attention_group["input_activations"]["dynamic"] is False
- assert attention_group["input_activations"]["symmetric"] is True
+ # Attention config is stored in a dedicated top-level field (not config_groups).
+ attention_config = saved_config["quantization_config"]["attention_input_activations"]
+ assert attention_config is not None
+ assert attention_config["targets"] == [compressed_model.model.decoder.layers[0].self_attn.__class__.__name__]
+ assert attention_config["input_activations"]["num_bits"] == 8
+ assert attention_config["input_activations"]["type"] == "float"
+ assert attention_config["input_activations"]["strategy"] == "tensor"
+ assert attention_config["input_activations"]["dynamic"] is False
+ assert attention_config["input_activations"]["symmetric"] is True
assert saved_config["quantization_config"]["kv_cache_scheme"] is not None
quantization_config = transformers.AutoConfig.from_pretrained(
quantized_model_path, trust_remote_code=True
).quantization_config
config_groups = quantization_config["config_groups"]
- attention_group = None
- for group in config_groups.values():
- targets = group["targets"]
- if "Linear" not in targets:
- attention_group = group
- break
-
- assert attention_group is not None
- assert attention_group["targets"] == [compressed_model.model.decoder.layers[0].self_attn.__class__.__name__]
- assert attention_group["weights"] is None
- assert attention_group["input_activations"]["num_bits"] == 8
- assert attention_group["input_activations"]["type"] == "float"
- assert attention_group["input_activations"]["strategy"] == "tensor"
- assert attention_group["input_activations"]["dynamic"] is False
- assert attention_group["input_activations"]["symmetric"] is True
+ # Only Linear layers should be in config_groups.
+ for group_targets in (g["targets"] for g in config_groups.values()):
+ assert "Linear" in group_targets, f"Unexpected non-Linear targets in config_groups: {group_targets}"
+
assert quantization_config["kv_cache_scheme"] is not None
assert getattr(compressed_model.model.decoder.layers[0].self_attn, "q_scale", None) is not None
@@ -293,46 +276,51 @@ def test_llmcompressor_static_fp8_attention_config(dataloader, tmp_path):
with open(os.path.join(quantized_model_path, "config.json")) as f:
saved_config = json.load(f)
- saved_groups = saved_config["quantization_config"]["config_groups"]
- attention_group = None
- for group in saved_groups.values():
- if "Linear" not in group["targets"]:
- attention_group = group
- break
-
- assert attention_group is not None
- assert attention_group["weights"] is None
- assert attention_group["input_activations"]["num_bits"] == 8
- assert attention_group["input_activations"]["type"] == "float"
- assert attention_group["input_activations"]["strategy"] == "tensor"
- assert attention_group["input_activations"]["dynamic"] is False
- assert attention_group["input_activations"]["symmetric"] is True
+
+ # Attention config is stored in a dedicated top-level field (not config_groups)
+ # to prevent compressed_tensors from trying to compress attention modules at load time.
+ attention_config = saved_config["quantization_config"]["attention_input_activations"]
+ assert attention_config is not None
+ assert "Linear" not in attention_config["targets"]
+ assert attention_config["input_activations"]["num_bits"] == 8
+ assert attention_config["input_activations"]["type"] == "float"
+ assert attention_config["input_activations"]["strategy"] == "tensor"
+ assert attention_config["input_activations"]["dynamic"] is False
+ assert attention_config["input_activations"]["symmetric"] is True
assert saved_config["quantization_config"]["kv_cache_scheme"] is not None
+ # Verify the model can be loaded without crashing.
model = AutoModelForCausalLM.from_pretrained(quantized_model_path, torch_dtype="auto", trust_remote_code=True)
quantization_config = model.config.quantization_config
config = quantization_config.to_dict() if hasattr(quantization_config, "to_dict") else quantization_config
config_groups = config["config_groups"]
+ # Only Linear layers should be in config_groups; attention modules use the
+ # separate attention_input_activations field.
assert "group_0" in config_groups
- attention_group = None
- for group in config_groups.values():
- targets = group["targets"]
- if "Linear" not in targets:
- attention_group = group
- break
-
- assert attention_group is not None
- assert attention_group["targets"] == [model.model.layers[0].self_attn.__class__.__name__]
- assert attention_group["weights"] is None
- assert attention_group["input_activations"]["num_bits"] == 8
- assert attention_group["input_activations"]["type"] == "float"
- assert attention_group["input_activations"]["strategy"] == "tensor"
- assert attention_group["input_activations"]["dynamic"] is False
- assert attention_group["input_activations"]["symmetric"] is True
- assert getattr(model.model.layers[0].self_attn, "q_scale", None) is not None
+ for group_targets in (g["targets"] for g in config_groups.values()):
+ assert "Linear" in group_targets, f"Unexpected non-Linear targets in config_groups: {group_targets}"
+
assert config["kv_cache_scheme"] is not None
+ # Verify KV cache quant parameters (q_scale, k_scale, v_scale) were saved
+ # in the checkpoint. They show as UNEXPECTED on the loaded model because the
+ # base HuggingFace architecture does not register them as standard parameters.
+ import glob as _glob
+
+ safetensors_files = _glob.glob(os.path.join(quantized_model_path, "*.safetensors"))
+ assert len(safetensors_files) > 0, "No safetensors file found"
+ from safetensors import safe_open as _safe_open
+
+ has_q_scale = False
+ for sf in safetensors_files:
+ with _safe_open(sf, framework="pt") as f:
+ for key in f.keys():
+ if "self_attn.q_scale" in key:
+ has_q_scale = True
+ break
+ assert has_q_scale, "q_scale not found in checkpoint"
+
def test_llmcompressor_mxfp8_export_packs_serially(tmp_path, monkeypatch):
autoround = AutoRound(