PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
[step 1]adjust gguf tuning algorithm (#1649)
Wenhua Cheng770b7a3a8dab
diff --git a/auto_round/alg_ext.py b/auto_round/alg_ext.pyindex 250aa8298..e6137d1a5 100644--- a/auto_round/alg_ext.py+++ b/auto_round/alg_ext.py@@ -68,7 +68,7 @@ def get_abs_top_percent_mask(x: torch.Tensor, percent: float = 1.0):inv_mask (torch.BoolTensor): Inverse of mask."""flat = x.view(-1)- k = max(1, int(flat.numel() * percent / 1000)) # 至少选1个+ k = max(1, int(flat.numel() * percent / 1000))_, idx = torch.topk(torch.abs(flat), k)mask = torch.zeros_like(flat, dtype=torch.bool)@@ -118,25 +118,14 @@ def quant_tensor_sym(init_scale=None,**kwargs,):- """Quantize and de-quantize tensor asymmetrically. full range, credit goes to llamacpp community+ """Quantize and de-quantize tensor symmetrically (full-range, llama.cpp style).- Args:- tensor: Tensor containing the tensor to be quantized- bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- group_size: Number of elements to share scale for quantization- v: Rounding value perturbation- min_scale: Minimum scale coefficient for tensor- max_scale: Maximum scale coefficient for tensor- tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.- tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.- scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32- q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability-- Returns:- Quantized and de-quantized tensor, scale, zero-point+ ``maxq`` is computed via ``int(2.0 ** (bits - 1))`` so it stays a plain+ Python int constant inside the inductor graph and Triton never tries to+ lower a ``2 ** SymInt`` through ``libdevice.pow(fp32, i64)``."""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** (bits - 1)+ maxq = int(2.0 ** (bits - 1))scale = init_scale * max_scale.unsqueeze(dim=-1)int_w = round_ste(tensor / scale + v)q = torch.clamp(int_w, -maxq, maxq - 1)@@ -149,7 +138,7 @@ def quant_tensor_sym(def qdq_mxfp(tensor, max_val, max_norm, emax, ebits, mbits):shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), torch.log2(max_val))shared_exp = torch.floor(shared_exp)- scale_emax = 2 ** (8 - 1) - 1+ scale_emax = (1 << (8 - 1)) - 1shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)scale = torch.pow(2.0, shared_exp)@@ -264,7 +253,7 @@ def quant_mx(# shared_exp = torch.log2(shared_exp + FP32_MIN_NORMAL * (shared_exp == 0).type(shared_exp.dtype))shared_exp = torch.where(max_val == 0, torch.ones_like(max_val), torch.log2(max_val))shared_exp = floor_ste(shared_exp)- scale_emax = 2 ** (8 - 1) - 1+ scale_emax = (1 << (8 - 1)) - 1shared_exp = (shared_exp - emax).clamp(min=-scale_emax, max=scale_emax)scale = torch.pow(2.0, shared_exp)@@ -523,443 +512,55 @@ def _register_act_max_hook_ext(self, model):return hook_handles-# ---------------------------- gguf alg -----------------------------from auto_round.data_type.gguf import double_quant_tensor-from auto_round.export.export_to_gguf.packing import make_qx_quants---def make_qp_quants(nmax, data, quant_weights, v=0):- data = data.to(torch.float32)- quant_weights = quant_weights.to(torch.float32)- group_max = torch.max(data, dim=-1, keepdim=True)[0]- scale = group_max / nmax- iscale = get_reciprocal(scale)- if isinstance(v, torch.Tensor) and v.numel() != 1:- v = v.view(data.shape)- v = v.to(data.device)-- L = torch.round(iscale * data + v)- diffs = data - scale * L- best_mse = torch.sum(quant_weights * diffs * diffs, dim=-1)-- for _is in range(-9, 10):- if _is == 0:- continue- scale_is = group_max / (0.1 * _is + nmax)- iscale_is = get_reciprocal(scale_is)-- tmp_L = torch.round(iscale_is * data + v).clip(max=nmax)- diffs = data - scale_is * tmp_L- mse = torch.sum(quant_weights * diffs * diffs, dim=-1)-- replace_idx = mse < best_mse- best_mse[replace_idx] = mse[replace_idx]- iscale[replace_idx] = iscale_is[replace_idx]-- L = torch.round(iscale * data + v).clip(max=nmax)- sumlx = torch.sum(quant_weights * data * L, dim=-1)- suml2 = torch.sum(quant_weights * L * L, dim=-1)- # When suml2 is zero (all L=0 or all quant_weights=0), fall back to the- # simple max-based scale estimate to avoid NaN propagating into the GGUF file.- fallback_d = group_max.squeeze(-1) / nmax- return torch.where(suml2 > 0, sumlx / suml2, fallback_d), L---# @torch._disable_dynamo()-def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, v=0):- """Adapted from Llamacpp. Performs iterative weighted least squares quantization search.-- Args:- data (torch.Tensor): Input tensor to quantize.- bits (int): Number of quantization bits.- rrmin (float): Initial range scaling factor.- rdelta (float): Step size for range scaling.- nstep (int): Number of search steps.- use_mad (bool): Whether to use mean absolute deviation instead of squared error.- weights (torch.Tensor): Weight matrix for each element.-- Returns:- Tuple: (Optimal scale tensor, optimal minimum value tensor)- """- dtype = torch.float32- data = data.to(dtype)- maxq = 2**bits - 1- minq = 0- weights = 1.0 if weights is None else weights.to(dtype)-- rmin = torch.min(data, dim=1, keepdim=True)[0]- rmax = torch.max(data, dim=1, keepdim=True)[0]-- sum_w = torch.sum(weights, dim=1, keepdim=True)- sum_x = torch.sum(weights * data, dim=1, keepdim=True)-- # scale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))- scale = (rmax - rmin) / (maxq - minq)- if isinstance(v, torch.Tensor) and v.numel() > 1:- v = v.reshape(data.shape)-- iscale = get_reciprocal(scale)- # quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)- quant_data = torch.clamp(torch.round(iscale * (data - rmin) + v), minq, maxq)- diff = scale * quant_data + rmin - data-- best_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff**2, dim=1, keepdim=True)-- for is_ in range(nstep):- factor = rrmin + rdelta * is_ + maxq - minq- # iscale_new = factor / (rmax - rmin + 1e-8)- scale_new = (rmax - rmin) / factor- iscale_new = get_reciprocal(scale_new)- quant_data_new = torch.clamp(torch.round(iscale_new * (data - rmin)), minq, maxq)-- mul_weights_quant_data = weights * quant_data_new- sum_l = torch.sum(mul_weights_quant_data, dim=-1, keepdim=True)- sum_l2 = torch.sum(mul_weights_quant_data * quant_data_new, dim=-1, keepdim=True)- sum_xl = torch.sum(mul_weights_quant_data * data, dim=-1, keepdim=True)-- D = sum_w * sum_l2 - sum_l**2- this_scale = (sum_w * sum_xl - sum_x * sum_l) / D- this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D- this_min[this_min > 0] = 0- this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]- reverse_this_scale = get_reciprocal(this_scale)-- quant_data = torch.clamp(torch.round(reverse_this_scale * (data - this_min) + v), minq, maxq)- diff = this_scale * quant_data + this_min - data- # diff = this_scale * quant_data_new + this_min - data- mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff**2, dim=-1, keepdim=True)-- idx_to_replace = torch.where((mad < best_mad) & (D > 0))[0]- best_mad[idx_to_replace] = mad[idx_to_replace]- scale[idx_to_replace] = this_scale[idx_to_replace]- rmin[idx_to_replace] = this_min[idx_to_replace]-- return scale.to(torch.float32), -rmin.to(torch.float32)---def make_qp_new_quants(data, orig_scale, orig_mins, quant_weights, bits=4, super_bits=6, data_v=0, scale_v=0, min_v=0):- nmax = 2**super_bits - 1- maxq = 2**bits - 1- minq = 0- orig_scale = orig_scale.to(torch.float32)- quant_weights = quant_weights.to(torch.float32)- group_max = torch.max(orig_scale, dim=-1, keepdim=True)[0]- s_scale = group_max / nmax- i_sscale = get_reciprocal(s_scale)- if isinstance(scale_v, torch.Tensor):- if scale_v.numel() != 1:- scale_v = scale_v.view(orig_scale.shape)- scale_v = scale_v.to(data.device)- data_v = data_v.to(data.device)-- L_scale = torch.round(i_sscale * orig_scale + scale_v)- qdq_scale = L_scale * s_scale- id_scale = get_reciprocal(qdq_scale)- id_scale = id_scale.view(-1, 1)- orig_mins = orig_mins.view(-1, 1) if orig_mins is not None and isinstance(orig_mins, torch.Tensor) else orig_mins- quant_data = torch.clamp(torch.round(id_scale * (data - orig_mins) + data_v.to(data.device)), minq, maxq)- qdq_scale = qdq_scale.view(-1, 1)- diff = qdq_scale * quant_data + orig_mins - data- best_mse = torch.sum(quant_weights * diff * diff, dim=-1)- best_mse = best_mse.view(orig_scale.shape)- best_mse = torch.sum(best_mse, dim=-1)- for _is in range(-9, 10):- if _is == 0:- continue- scale_s_is = group_max / (0.1 * _is + nmax)- iscale_s_is = get_reciprocal(scale_s_is)-- tmp_L_scale = torch.round(iscale_s_is * orig_scale + scale_v).clip(min=0, max=nmax)- qdq_scale = scale_s_is * tmp_L_scale- reverse_this_scale = get_reciprocal(qdq_scale)- reverse_this_scale = reverse_this_scale.view(-1, 1)- quant_data = torch.clamp(torch.round(reverse_this_scale * (data - orig_mins) + data_v), minq, maxq)- diffs = qdq_scale.view(-1, 1) * quant_data + orig_mins - data- mse = torch.sum(quant_weights * diffs * diffs, dim=-1)- mse = mse.view(orig_scale.shape)- mse = torch.sum(mse, dim=-1)- replace_idx = mse < best_mse- best_mse[replace_idx] = mse[replace_idx]- i_sscale[replace_idx] = iscale_s_is[replace_idx]-- L = torch.round(i_sscale * orig_scale + scale_v).clip(max=nmax)- quant_weights = torch.sum(quant_weights, dim=-1)- quant_weights = quant_weights.view(orig_scale.shape)- sumlx = torch.sum(quant_weights * orig_scale * L, dim=-1)- suml2 = torch.sum(quant_weights * L * L, dim=-1)- # When suml2 is zero, fall back to the simple max-based scale estimate- # to avoid NaN propagating into the GGUF file.- fallback_d = group_max.squeeze(-1) / nmax- return torch.where(suml2 > 0, sumlx / suml2, fallback_d), L---def quant_tensor_gguf_asym_dq(- tensor,- bits=4,- v=0,- min_scale=1.0,- max_scale=1.0,- scale_dtype=torch.float16,- tensor_min=None,- tensor_max=None,- q_scale_thresh=1e-5,- imatrix=None,- prev_scale=None,- prev_wmin=None,- prev_d_scale=None,- prev_d_wmin=None,- iter=0,- scale_v=0,- wmin_v=0,- **kwargs,-):- """Quantizes and dequantizes a tensor using asymmetric integer quantization for formats like Q2_K, Q4_K, and Q5_K.- Only fit for iters 0-- Args:- tensor (torch.Tensor): Input tensor to quantize.- bits (int): Number of bits for quantization.- group_size (int): Group size for per-group quantization.- v (float): Perturbation added before rounding.- min_scale (float): Minimum allowed scale value.- max_scale (float): Maximum allowed scale value.- scale_dtype (torch.dtype): Data type for quantized scale.- tensor_min (torch.Tensor, optional): Minimum values for the tensor groups.- tensor_max (torch.Tensor, optional): Maximum values for the tensor groups.- q_scale_thresh (float): Threshold to clamp the quantized scale.- super_group_size (int): Number of groups to bundle for secondary quantization.- super_bits (int): Number of bits used in secondary quantization.- imatrix (torch.Tensor, optional): Importance matrix for weighted quantization.+def _dq_asym_qdq(tensor, scale, wmin, bits, group_size, v=0):+ """Pure asym double-quant qdq math given precomputed scale/wmin.- Returns:- Tuple: (Quantized-dequantized tensor, scale dictionary, zero-point dictionary)+ ``maxq`` is computed via ``int(2.0 ** bits) - 1`` so that any SymInt+ handling for ``bits`` does not produce a ``libdevice.pow(fp32, i64)`` call+ in Triton (which lacks that overload). The final ``int(...)`` cast keeps+ ``maxq`` as a Python int constant inside the compiled graph."""-- orig_dtype = tensor.dtype- maxq = 2**bits - 1- group_size = 16 if bits == 2 else 32- super_bits = 4 if bits == 2 else 6- super_group_size = 16 if bits == 2 else 8tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtypetensor = tensor.to(torch.float32)- if iter is None:- iter = 0- if iter % 10 == 0 or iter == -1 or prev_scale is None:- if bits not in [2, 4, 5]:- raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")- quant_weights = None- if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):- search_kwargs = {- 2: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": True},- 4: {"rmin": -1, "rdelta": 0.1, "nstep": 20, "use_mad": False},- 5: {"rmin": -0.5, "rdelta": 0.1, "nstep": 15, "use_mad": False},- }- if bits == 2:- quant_weights = torch.abs(tensor)- elif bits == 4 or bits == 5:- sigma2 = torch.sum(tensor**2, dim=-1, keepdim=True) / 32 ##Note 32 is different from QK_K- av_x = torch.sqrt(sigma2)- quant_weights = torch.abs(tensor) + av_x- params = search_kwargs[bits]- scale, wmin = iterative_wls_quant_search(- tensor,- bits=bits,- rrmin=params["rmin"],- rdelta=params["rdelta"],- nstep=params["nstep"],- use_mad=params["use_mad"],- weights=quant_weights,- v=v,- )- scale = scale.to(scale_dtype)- scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)- scale = scale.reshape(-1, super_group_size)- wmin = wmin.reshape(-1, super_group_size)- scale, d_scale = double_quant_tensor(scale, super_bits)- wmin = torch.where(torch.abs(wmin) < 1e-30, torch.zeros_like(wmin), wmin)- wmin, d_wmin = double_quant_tensor(wmin, super_bits)- wmin = wmin.view(-1, 1)- scale = scale.view(-1, 1)- else:- imatrix = imatrix.to(tensor.device)- search_kwargs = {- 2: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},- 4: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},- 5: {"rmin": -0.9, "rdelta": 0.05, "nstep": 36, "use_mad": False},- }-- weights = imatrix.reshape(1, -1)-- weights = weights.expand(tensor.numel() // weights.numel(), -1)- quant_weights = weights.reshape(tensor.shape)-- if torch.min(quant_weights) == 0:- logger.warning_once(- "please use more data via setting `nsamples` "- "to improve accuracy as calibration activations contain 0"- )-- zero_cnt = torch.sum(quant_weights == 0, dim=-1)- replace_index = zero_cnt > group_size // 2- if torch.sum(replace_index) > 0:- # Fallback to no imatrix- if bits == 2:- tmp_quant_weights = torch.abs(tensor)- elif bits == 4 or bits == 5:- sigma2 = torch.sum(tensor**2, dim=-1, keepdim=True) / 32 ## Note 32 is different from QK_K- av_x = torch.sqrt(sigma2)- tmp_quant_weights = torch.abs(tensor) + av_x- quant_weights[replace_index, :] = tmp_quant_weights[replace_index, :]- mean_replace_index = (zero_cnt > 0) & (zero_cnt <= group_size // 2)- if torch.sum(mean_replace_index) > 0:- ## use mean values to fill zero values- tmp_quant_weights = torch.sum(quant_weights, dim=-1) / (quant_weights.shape[1] - zero_cnt)- tmp_quant_weights = tmp_quant_weights.view(-1, 1).expand(-1, quant_weights.shape[1])- quant_weights[mean_replace_index, :] = tmp_quant_weights[mean_replace_index, :]-- params = search_kwargs[bits]-- scale, wmin_0 = iterative_wls_quant_search(- tensor,- bits=bits,- rrmin=params["rmin"],- rdelta=params["rdelta"],- nstep=params["nstep"],- use_mad=params["use_mad"],- weights=quant_weights,- v=v,- )- scale = scale.to(scale_dtype)- scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)- nmax = 2**super_bits - 1- scale = scale.reshape(-1, super_group_size)- wmin = wmin_0.reshape(-1, super_group_size)- sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)-- d_scale, q_scale = make_qp_new_quants(tensor, scale, wmin, quant_weights, bits, super_bits, data_v=v)- d_scale = d_scale.unsqueeze(-1)-- d_wmin, q_wmin = make_qp_quants(nmax, wmin, sum_quant_weights, v=wmin_v)-- d_wmin = d_wmin.unsqueeze(-1)- scale = (d_scale * q_scale).view(-1, 1)- wmin = (d_wmin * q_wmin).view(-1, 1)- else:- scale = prev_scale.detach()- d_scale = prev_d_scale.detach()- wmin = prev_wmin.detach()- d_wmin = prev_d_wmin.detach()+ maxq = int(2.0**bits) - 1inverse_scale = get_reciprocal(scale)-int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)- qdq_result = (scale * int_w - wmin).to(orig_dtype)- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}+ qdq = (scale * int_w - wmin).to(orig_dtype)+ qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)+ return qdq-def quant_tensor_gguf_sym_dq(- tensor,- bits=3,- v=0,- imatrix=None,- prev_scale=None,- prev_d_scale=None,- iter=0,- **kwargs,-):- """Quantize and de-quantize tensor asymmetrically. For Q3_K, Q6_K.+def _dq_sym_qdq(tensor, scale, bits, v=0):+ """Pure sym double-quant qdq math given precomputed scale.- Args:- tensor: Tensor containing the tensor to be quantized- bits: Number of bits for quantization (e.g., 2, 3, 4, 8)- group_size: Number of elements to share scale for quantization- v: Rounding value perturbation- min_scale: Minimum scale coefficient for tensor- max_scale: Maximum scale coefficient for tensor- tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.- tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.- scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import- q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability-- Returns:- Quantized and de-quantized tensor, scale, zero-point+ ``maxq`` is computed via float ``2.0 ** (bits - 1)`` then cast to+ ``int`` to avoid SymInt-driven shifts being lowered through+ ``libdevice.pow``."""- from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, K_SCALE_SIZE, QK_K+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K- if bits not in [3, 6]:- raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")-- maxq = 2 ** (bits - 1)group_size = 16- super_bits = 6 if bits == 3 else 8super_group_size = 16-+ maxq = int(2.0 ** (bits - 1))tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- ggml_type = f"q{bits}_k"- block_size, type_size = GGML_QUANT_SIZES[ggml_type]orig_dtype = tensor.dtype-tensor = tensor.to(torch.float32)+ ggml_type = f"q{bits}_k"+ block_size, _ = GGML_QUANT_SIZES[ggml_type]n_blocks = tensor.nelement() // block_size- # (nb, 16, 16)- # tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)-- if iter is None:- iter = 0- if iter % 10 == 0 or iter == -1 or prev_scale is None:- if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):- if bits == 3:- from auto_round.export.export_to_gguf.packing import make_q3_quants-- scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True)- elif bits == 6:- scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)- else:- imatrix = imatrix.to(tensor.device)-- weights = imatrix.reshape(1, -1)- weights = weights.expand(tensor.numel() // weights.numel(), -1)- quant_weights = weights.reshape(tensor.shape)- if torch.min(quant_weights) == 0:- logger.warning_once(- "please use more data via setting `nsamples` "- "to improve accuracy as calibration activations contain 0"- )- zero_cnt = torch.sum(quant_weights == 0, dim=-1)- replace_index = zero_cnt > group_size // 2- if torch.sum(replace_index) > 0:- if bits == 6:- quant_weights[replace_index] = tensor[replace_index] * tensor[replace_index]- else:- sigma2 = 2 * torch.sum(torch.pow(tensor, 2), dim=-1, keepdim=True) / QK_K- tmp_quant_weights = torch.sqrt(sigma2 + tensor * tensor)- quant_weights[replace_index] = tmp_quant_weights[replace_index]- mean_replace_index = (zero_cnt > 0) & (zero_cnt <= group_size // 2)- if torch.sum(mean_replace_index) > 0:- ## use mean values to fill zero values- tmp_quant_weights = torch.sum(quant_weights, dim=-1) / (quant_weights.shape[-1] - zero_cnt)- tmp_quant_weights = (- tmp_quant_weights.view(-1, 1).expand(-1, quant_weights.shape[1]).reshape(tensor.shape)- )- quant_weights[mean_replace_index] = tmp_quant_weights[mean_replace_index]-- # scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights, v=v)- scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=quant_weights)- scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)- # conduct double quant- d_scale, q_scale = make_qp_new_quants(tensor, scale, 0, quant_weights, bits, super_bits, data_v=v)- d_scale = d_scale.unsqueeze(-1)- scale = (d_scale * q_scale).unsqueeze(-1)+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ if isinstance(v, torch.Tensor):+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)+ v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)else:- scale = prev_scale.detach()- d_scale = prev_d_scale.detach()- zp = torch.full_like(scale, maxq) # pylint: disable=E1130+ v_r = v+ zp = torch.full_like(scale, maxq)inverse_scale = get_reciprocal(scale)- int_w = round_ste(tensor * inverse_scale + v).clip(-maxq, maxq - 1) + maxq- qdq_result = (scale * (int_w - zp)).to(orig_dtype)- qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)-- return qdq_result, {"scale": scale, "d_scale": d_scale}, zp+ int_w = round_ste(tensor * inverse_scale + v_r).clip(-maxq, maxq - 1) + maxq+ qdq = (scale * (int_w - zp)).to(orig_dtype)+ qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)+ return qdqclass DQWrapperLinear(WrapperLinear):@@ -1017,10 +618,24 @@ class DQWrapperLinear(WrapperLinear):"""super()._init_tuning_params_and_quant_func()p_dtype = torch.float32+ # ``search_func`` is kept un-compiled because it contains data-dependent+ # control flow (imatrix branches, iterative searches), while+ # ``weight_quant_func`` is the compilable pure-math part.+ self.search_func = None+ self._dq_kind = None+ self._is_dq_path = Falseif hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:- self.weight_quant_func = (- quant_tensor_gguf_asym_dq if self.orig_layer.data_type == "int_asym_dq" else quant_tensor_gguf_sym_dq- )+ self._is_dq_path = True+ from auto_round.data_type.gguf import search_gguf_scale_min_asym, search_gguf_scale_min_sym++ if self.orig_layer.data_type == "int_asym_dq":+ self.search_func = search_gguf_scale_min_asym+ self.weight_quant_func = _dq_asym_qdq+ self._dq_kind = "asym"+ else:+ self.search_func = search_gguf_scale_min_sym+ self.weight_quant_func = _dq_sym_qdq+ self._dq_kind = "sym"elif self.orig_layer.sym:from auto_round.data_type.int import quant_tensor_sym@@ -1031,11 +646,10 @@ class DQWrapperLinear(WrapperLinear):self.weight_quant_func = quant_tensor_asymself.data_type = self.orig_layer.data_typeif self.enable_act_quant:- self.act_quant_func = (- quant_tensor_gguf_asym_dq- if self.orig_layer.act_data_type == "int_asym_dq"- else quant_tensor_gguf_sym_dq- )+ from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq as _gguf_asym_dq+ from auto_round.data_type.gguf import quant_tensor_gguf_sym_dq as _gguf_sym_dq++ self.act_quant_func = _gguf_asym_dq if self.orig_layer.act_data_type == "int_asym_dq" else _gguf_sym_dqif self.enable_torch_compile:self.act_quant_func = compile_func(self.act_quant_func, self.device)self._init_params("act_max_scale", p_dtype, (1), 1.0, not self.orig_layer.act_dynamic)@@ -1043,7 +657,67 @@ class DQWrapperLinear(WrapperLinear):if self.enable_torch_compile:self.weight_quant_func = compile_func(self.weight_quant_func, self.device)- def _qdq_weight(self, value, min_scale, max_scale, scale_v=None, wmin_v=None, iter=None):+ @torch.no_grad()+ def _run_search(self, weight, v):+ """Run the per-format scale/wmin search separately from the quant func.++ Uses the search routines from ``auto_round.data_type.gguf`` and forwards+ the tuning perturbation ``v``. Returns the parameters to feed into the+ (compilable) ``weight_quant_func``.+ """+ from auto_round.data_type.gguf import double_quant_tensor_sym_rtn+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K++ bits = self.orig_layer.bits+ scale_dtype = self.orig_layer.scale_dtype+ imatrix = getattr(self.orig_layer, "imatrix", None)++ if self._dq_kind == "asym":+ group_size = 16 if bits == 2 else 32+ t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)+ v_r = v+ if isinstance(v, torch.Tensor):+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)+ scale, wmin, d_scale, d_wmin = self.search_func(+ t,+ bits=bits,+ scale_dtype=scale_dtype,+ imatrix=imatrix,+ split_num=1,+ v=v_r,+ )+ # Search funcs are decorated with ``@torch.inference_mode()``; their+ # outputs are inference tensors and cannot be saved for backward.+ # Clone to detach from inference mode so autograd may use them.+ return {+ "scale": scale.clone(),+ "wmin": wmin.clone(),+ "d_scale": d_scale.clone(),+ "d_wmin": d_wmin.clone(),+ }++ # sym path+ group_size = 16+ super_group_size = 16+ t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)+ ggml_type = f"q{bits}_k"+ block_size, _ = GGML_QUANT_SIZES[ggml_type]+ n_blocks = t.nelement() // block_size+ t = t.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ v_r = v+ if isinstance(v, torch.Tensor):+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)+ v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ super_bits = 6 if bits == 3 else 8+ scale = self.search_func(t, bits, imatrix, scale_dtype, split_num=1, v=v_r)+ scale = scale.to(scale_dtype)+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)+ scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)+ scale = scale.unsqueeze(-1)+ # Clone to escape inference-mode tensors (see asym branch comment).+ return {"scale": scale.clone(), "d_scale": d_scale.clone()}++ def _qdq_weight(self, value, min_scale, max_scale, scale_v=None, iter=None):"""Quantizes and dequantizes weights with tuning parameters.Args:@@ -1064,6 +738,57 @@ class DQWrapperLinear(WrapperLinear):if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):weight = weight.t()+ if self._is_dq_path:+ # Split search (data-dependent, un-compiled) from quant math (compilable).+ iter_v = self.cur_iter if (iter is None and hasattr(self, "cur_iter")) else iter+ if iter_v is None:+ iter_v = 0+ need_search = (iter_v % 10 == 0) or (iter_v == -1) or (self.prev_scale is None)+ if need_search:+ params = self._run_search(weight, value)+ self.prev_scale = params["scale"]+ self.prev_d_scale = params["d_scale"]+ if self._dq_kind == "asym":+ self.prev_wmin = params["wmin"]+ self.prev_d_wmin = params["d_wmin"]+ else:+ params = {+ "scale": self.prev_scale.detach(),+ "d_scale": self.prev_d_scale.detach(),+ }+ if self._dq_kind == "asym":+ params["wmin"] = self.prev_wmin.detach()+ params["d_wmin"] = self.prev_d_wmin.detach()++ bits = self.orig_layer.bits+ if self._dq_kind == "asym":+ group_size = 16 if bits == 2 else 32+ weight_q = self.weight_quant_func(+ weight,+ params["scale"],+ params["wmin"],+ bits,+ group_size,+ v=value,+ )+ scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}+ zp_out = {"wmin": params["wmin"], "d_wmin": params["d_wmin"]}+ else:+ weight_q = self.weight_quant_func(+ weight,+ params["scale"],+ bits,+ v=value,+ )+ scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}+ zp_out = torch.full_like(params["scale"], int(2.0 ** (bits - 1)))++ weight_q = weight_q.to(weight.dtype)+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):+ weight_q = weight_q.t()+ return weight_q, scale_out, zp_out++ # Non-dq path: preserve original behavior.quant_kwargs = {}if hasattr(self.orig_layer, "super_bits"):quant_kwargs["super_bits"] = self.orig_layer.super_bits@@ -1084,11 +809,6 @@ class DQWrapperLinear(WrapperLinear):prev_wmin=self.prev_wmin,prev_d_scale=self.prev_d_scale,prev_d_wmin=self.prev_d_wmin,- imatrix=self.orig_layer.imatrix if hasattr(self.orig_layer, "imatrix") else None,- iter=self.cur_iter if (iter is None and hasattr(self, "cur_iter")) else iter,- # scale_v=self.scale_v if scale_v is None else scale_v,- # wmin_v=self.wmin_v if wmin_v is None else wmin_v,- # xtx=self.orig_layer.xtx if hasattr(self.orig_layer, "xtx") else None,**quant_kwargs,)weight_q = weight_q.to(weight.dtype)diff --git a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py b/auto_round/algorithms/quantization/sign_roundv2/quantizer.pyindex 36e5ab9dc..145215dd1 100644--- a/auto_round/algorithms/quantization/sign_roundv2/quantizer.py+++ b/auto_round/algorithms/quantization/sign_roundv2/quantizer.py@@ -22,16 +22,66 @@ from torch import autocastfrom auto_round.algorithms.quantization.sign_round.config import SignRoundConfigfrom auto_round.algorithms.quantization.sign_round.quantizer import SignRoundQuantizer-from auto_round.data_type.gguf import quant_tensor_gguf_asym_dq, quant_tensor_gguf_sym_dq+from auto_round.data_type.gguf import (+ double_quant_tensor_sym_rtn,+ quant_tensor_gguf_asym_dq,+ quant_tensor_gguf_sym_dq,+ search_gguf_scale_min_asym,+ search_gguf_scale_min_sym,+)from auto_round.data_type.int import quant_tensor_asym, quant_tensor_sym, search_scalesfrom auto_round.data_type.mxfp import quant_mx, search_mx_scalefrom auto_round.data_type.nvfp import nv_fp4, search_nvfp4_scale-from auto_round.data_type.utils import reshape_pad_tensor_by_group_size+from auto_round.data_type.utils import (+ reshape_pad_tensor_by_group_size,+ revert_tensor_by_pad,+ round_ste,+)from auto_round.logger import logger-from auto_round.utils import check_to_quantized, compile_func+from auto_round.utils import check_to_quantized, compile_func, get_reciprocalfrom auto_round.wrapper import WrapperLinear, wrapper_block+def _dq_asym_qdq(tensor, scale, wmin, bits, group_size, v=0):+ """Pure asym double-quant qdq math given precomputed scale/wmin (compilable)."""+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ tensor = tensor.to(torch.float32)+ maxq = int(2.0**bits) - 1+ inverse_scale = get_reciprocal(scale)+ int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)+ qdq = (scale * int_w - wmin).to(orig_dtype)+ qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)+ return qdq+++def _dq_sym_qdq(tensor, scale, bits, v=0):+ """Pure sym double-quant qdq math given precomputed scale (compilable)."""+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K++ group_size = 16+ super_group_size = 16+ maxq = int(2.0 ** (bits - 1))+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ tensor = tensor.to(torch.float32)+ ggml_type = f"q{bits}_k"+ block_size, _ = GGML_QUANT_SIZES[ggml_type]+ n_blocks = tensor.nelement() // block_size+ tensor = tensor.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ if isinstance(v, torch.Tensor):+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)+ v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ else:+ v_r = v+ zp = torch.full_like(scale, maxq)+ inverse_scale = get_reciprocal(scale)+ int_w = round_ste(tensor * inverse_scale + v_r).clip(-maxq, maxq - 1) + maxq+ qdq = (scale * (int_w - zp)).to(orig_dtype)+ qdq = revert_tensor_by_pad(qdq, orig_shape=orig_shape, pad_len=pad_len)+ return qdq++def _named_wrapper_block(wrapper_cls, name: str):wrapped = partial(wrapper_block, wrapper_cls=wrapper_cls)wrapped.__name__ = name@@ -84,6 +134,9 @@ class SignRoundDQWrapperLinear(WrapperLinear):minmax_scale_bound = (0.5, 1.5)def __init__(self, *args, **kwargs):+ if "enable_minmax_tuning" in kwargs:+ logger.warning_once("disable minmax tuning for a little better accuracy and lower cost")+ kwargs["enable_minmax_tuning"] = False # a little faster and bettersuper().__init__(*args, **kwargs)self.prev_scale = Noneself.prev_wmin = None@@ -92,10 +145,21 @@ class SignRoundDQWrapperLinear(WrapperLinear):def _init_tuning_params_and_quant_func(self):super()._init_tuning_params_and_quant_func()+ # The double-quant search path is data-dependent and kept un-compiled,+ # while ``weight_quant_func`` is the compilable pure-math half.+ self._is_dq_path = False+ self._dq_kind = None+ self.search_func = Noneif hasattr(self.orig_layer, "super_group_size") and self.orig_layer.super_group_size is not None:- self.weight_quant_func = (- quant_tensor_gguf_asym_dq if self.orig_layer.data_type == "int_asym_dq" else quant_tensor_gguf_sym_dq- )+ self._is_dq_path = True+ if self.orig_layer.data_type == "int_asym_dq":+ self.search_func = search_gguf_scale_min_asym+ self.weight_quant_func = _dq_asym_qdq+ self._dq_kind = "asym"+ else:+ self.search_func = search_gguf_scale_min_sym+ self.weight_quant_func = _dq_sym_qdq+ self._dq_kind = "sym"elif self.orig_layer.sym:self.weight_quant_func = quant_tensor_symelse:@@ -113,26 +177,120 @@ class SignRoundDQWrapperLinear(WrapperLinear):if self.enable_torch_compile:self.weight_quant_func = compile_func(self.weight_quant_func, self.device)+ @torch.no_grad()+ def _run_search(self, weight, v):+ """Per-format scale/wmin search separated from the (compilable) quant func."""+ from auto_round.export.export_to_gguf.config import GGML_QUANT_SIZES, QK_K++ bits = self.orig_layer.bits+ scale_dtype = self.orig_layer.scale_dtype+ imatrix = getattr(self.orig_layer, "imatrix", None)++ if self._dq_kind == "asym":+ group_size = 16 if bits == 2 else 32+ t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)+ v_r = v+ if isinstance(v, torch.Tensor):+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)+ scale, wmin, d_scale, d_wmin = self.search_func(+ t,+ bits=bits,+ scale_dtype=scale_dtype,+ imatrix=imatrix,+ split_num=1,+ v=v_r,+ )+ # Search funcs use ``@torch.inference_mode()``; clone to detach so+ # autograd may consume them.+ return {+ "scale": scale.clone(),+ "wmin": wmin.clone(),+ "d_scale": d_scale.clone(),+ "d_wmin": d_wmin.clone(),+ }++ # sym path+ group_size = 16+ super_group_size = 16+ t, _, _ = reshape_pad_tensor_by_group_size(weight.to(torch.float32), group_size)+ ggml_type = f"q{bits}_k"+ block_size, _ = GGML_QUANT_SIZES[ggml_type]+ n_blocks = t.nelement() // block_size+ t = t.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ v_r = v+ if isinstance(v, torch.Tensor):+ v_r, _, _ = reshape_pad_tensor_by_group_size(v, group_size)+ v_r = v_r.reshape(n_blocks, super_group_size, QK_K // super_group_size)+ super_bits = 6 if bits == 3 else 8+ scale = self.search_func(t, bits, imatrix, scale_dtype, split_num=1, v=v_r)+ scale = scale.to(scale_dtype)+ scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)+ scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)+ scale = scale.unsqueeze(-1)+ return {"scale": scale.clone(), "d_scale": d_scale.clone()}+def _qdq_weight(self, value, min_scale, max_scale):- weight_q, scale, zp = super()._qdq_weight(value, min_scale, max_scale)- if isinstance(scale, dict) and "d_scale" in scale and self.prev_scale is None:- self.prev_scale = scale["scale"]- self.prev_d_scale = scale["d_scale"]- if isinstance(zp, dict):- self.prev_wmin = zp["wmin"]- self.prev_d_wmin = zp["d_wmin"]- elif self.prev_scale is None:- self.prev_scale = scale- return weight_q, scale, zp-- def _extra_quant_kwargs(self):- return {- "prev_scale": self.prev_scale,- "prev_wmin": self.prev_wmin,- "prev_d_scale": self.prev_d_scale,- "prev_d_wmin": self.prev_d_wmin,- "iter": getattr(self, "cur_iter", None),- }+ if not self._is_dq_path:+ # Non-dq path keeps the original behavior (base class handles it).+ return super()._qdq_weight(value, min_scale, max_scale)++ if self.orig_layer.bits >= 16:+ return self.orig_layer.weight, None, None+ min_bound, max_bound = self.minmax_scale_bound+ min_scale.data.clamp_(min_bound, max_bound) # TODO this one could be deleted+ max_scale.data.clamp_(min_bound, max_bound)+ weight = self.orig_layer.weight+ if weight.device.type == "meta":+ weight = self.orig_layer.get_weight().to(self.device)+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):+ weight = weight.t()++ # Re-search every 10 steps; otherwise reuse the cached search results.+ iter_v = getattr(self, "cur_iter", 0)+ need_search = (iter_v == 0) or (iter_v == -1) or (self.prev_scale is None)+ if need_search:+ params = self._run_search(weight, value)+ self.prev_scale = params["scale"]+ self.prev_d_scale = params["d_scale"]+ if self._dq_kind == "asym":+ self.prev_wmin = params["wmin"]+ self.prev_d_wmin = params["d_wmin"]+ else:+ params = {+ "scale": self.prev_scale.detach(),+ "d_scale": self.prev_d_scale.detach(),+ }+ if self._dq_kind == "asym":+ params["wmin"] = self.prev_wmin.detach()+ params["d_wmin"] = self.prev_d_wmin.detach()++ bits = self.orig_layer.bits+ if self._dq_kind == "asym":+ group_size = 16 if bits == 2 else 32+ weight_q = self.weight_quant_func(+ weight,+ params["scale"],+ params["wmin"],+ bits,+ group_size,+ v=value,+ )+ scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}+ zp_out = {"wmin": params["wmin"], "d_wmin": params["d_wmin"]}+ else:+ weight_q = self.weight_quant_func(+ weight,+ params["scale"],+ bits,+ v=value,+ )+ scale_out = {"scale": params["scale"], "d_scale": params["d_scale"]}+ zp_out = torch.full_like(params["scale"], int(2.0 ** (bits - 1)))++ weight_q = weight_q.to(weight.dtype)+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):+ weight_q = weight_q.t()+ return weight_q, scale_out, zp_outclass SignRoundV2Quantizer(SignRoundQuantizer):@@ -153,7 +311,10 @@ class SignRoundV2Quantizer(SignRoundQuantizer):"algorithm extension has only undergone limited validation on ""W2A16,INT4, MXFP4 and NVFP4; use with caution.")- self._use_outlier_suppressed_loss = True+ if self.act_bits <= 4 or self.bits < 4:+ self._use_outlier_suppressed_loss = True+ else:+ self._use_outlier_suppressed_loss = Falseself.wrapper_block = _named_wrapper_block(SignRoundOptimizedWrapperLinear, "wrapper_block")if self.data_type.endswith("dq"):diff --git a/auto_round/compressors/base.py b/auto_round/compressors/base.pyindex 65e2158b2..af36c0324 100644--- a/auto_round/compressors/base.py+++ b/auto_round/compressors/base.py@@ -662,13 +662,13 @@ class BaseCompressor(object):if self.enable_torch_compile and is_raw_nv_fp:self.enable_torch_compile = Falselogger.warning_once("reset enable_torch_compile to `False` as nvfp4 is enabled")- super_group_size = getattr(cfg, "super_group_size", None)- enable_alg_ext = getattr(cfg, "enable_alg_ext", False)- if self.enable_torch_compile and super_group_size is not None and enable_alg_ext:- self.enable_torch_compile = False- logger.warning_once(- "reset enable_torch_compile to `False` as super_group_size is set for algorithm extension"- )+ # super_group_size = getattr(cfg, "super_group_size", None)+ # enable_alg_ext = getattr(cfg, "enable_alg_ext", False)+ # if self.enable_torch_compile and super_group_size is not None and enable_alg_ext:+ # self.enable_torch_compile = False+ # logger.warning_once(+ # "reset enable_torch_compile to `False` as super_group_size is set for algorithm extension"+ # )def _get_calibration_dataset(self) -> str:"""Resolve calibration dataset: self.dataset > AutoScheme.dataset > default."""diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 6541ad0de..7fea96c9a 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -59,7 +59,7 @@ def quant_tensor_sym_dq("""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** (bits - 1)+ maxq = int(2.0 ** (bits - 1))if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -78,7 +78,7 @@ def quant_tensor_sym_dq(scale = scale.view(-1, 1)zp = torch.full_like(scale, maxq) # pylint: disable=E1130int_w = round_ste(tensor * get_reciprocal(scale) + v)- q = torch.clamp(int_w + zp, 0, 2**bits - 1)+ q = torch.clamp(int_w + zp, 0, int(2.0**bits) - 1)qdq_result = (scale * (q - zp)).to(tensor.dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)return qdq_result, {"scale": scale, "d_scale": d_scale}, zp@@ -116,7 +116,7 @@ def quant_tensor_asym_float_zp(Quantized and de-quantized tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2**bits - 1+ maxq = int(2.0**bits) - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -141,10 +141,69 @@ def quant_tensor_asym_float_zp(return qdq_result, scale, zp+@register_dtype("opt_rtn_int_asym_float_zp")+def quant_tensor_asym_float_zp_rtn(+ tensor,+ bits=4,+ group_size=-1,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs,+):+ """RTN (round-to-nearest) version of asymmetric float-zp quantization.++ Optimized with in-place operations to minimize temporary allocations.+ Does not support per-element perturbation ``v`` or ``min_scale``/``max_scale``+ tensor scaling (pure RTN).++ Args:+ tensor: Tensor to quantize.+ bits: Number of bits for quantization.+ group_size: Number of elements sharing one scale.+ scale_dtype: dtype of the quantized scale.+ tensor_min/tensor_max: Optional pre-computed per-group min/max.+ q_scale_thresh: Lower clip for scale magnitude.++ Returns:+ Tuple of (qdq tensor, scale, zero-point).+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ orig_dtype = tensor.dtype+ maxq = int(2.0**bits) - 1++ if tensor_min is None or tensor_max is None:+ wmin = torch.clamp_max(tensor.min(-1)[0], 0)+ wmax = torch.clamp_min(tensor.max(-1)[0], 0)+ else:+ wmin = tensor_min+ wmax = tensor_max++ # scale = (wmax - wmin) / maxq+ scale = (wmax - wmin).div_(maxq).to(scale_dtype)+ scale.clamp_(min=q_scale_thresh)++ # zp = -wmin / scale+ zp = (-wmin).div_(scale)++ scale = scale.unsqueeze(-1)+ zp = zp.unsqueeze(-1)++ # Cast to float32 for stable inplace math, then back to original dtype+ if tensor.dtype != torch.float32:+ tensor = tensor.float()+ inverse_scale = get_reciprocal(scale)+ tensor.mul_(inverse_scale).round_().add_(zp).clamp_(0, maxq).sub_(zp).mul_(scale)+ tensor = tensor.to(orig_dtype)+ tensor = revert_tensor_by_pad(tensor, orig_shape=orig_shape, pad_len=pad_len)+ return tensor, scale, zp++## the values should be positivedef double_quant_tensor(tensor, bits):tensor = tensor.to(torch.float32) # Ensure tensor is in float32 for precision- maxq = 2**bits - 1+ maxq = int(2.0**bits) - 1wmax = torch.clamp(tensor.max(-1)[0], min=0)scale = wmax / maxqscale = scale.view(-1, 1)@@ -156,7 +215,7 @@ def double_quant_tensor(tensor, bits):def double_quant_tensor_sym(tensor, bits):tensor = tensor.to(torch.float32) # Ensure tensor is in float32 for precision- maxq = 2 ** (bits - 1)+ maxq = int(2.0 ** (bits - 1))imax = abs(tensor).argmax(axis=-1, keepdims=True)wmax = torch.take_along_dim(tensor, imax, dim=-1)scale = wmax / -maxq@@ -175,7 +234,7 @@ def double_quant_tensor_sym_rtn(tensor, bits):if tensor.dtype != torch.float32:tensor = tensor.float() # .float() creates a copy if needed- maxq = 2 ** (bits - 1)+ maxq = int(2.0 ** (bits - 1))# Compute absolute max along last dim# abs_() is inplace@@ -196,14 +255,14 @@ def double_quant_tensor_sym_rtn(tensor, bits):return tensor, scale-def make_qp_quants(nmax, data, quant_weights):+def make_qp_quants(nmax, data, quant_weights, v=0):data = data.to(torch.float32)quant_weights = quant_weights.to(torch.float32)group_max = torch.max(data, dim=-1, keepdim=True)[0]scale = group_max / nmaxiscale = get_reciprocal(scale)- L = torch.round(iscale * data)+ L = torch.round(iscale * data + v)diffs = data - scale * Lbest_mse = torch.sum(quant_weights * diffs * diffs, dim=-1)@@ -213,7 +272,7 @@ def make_qp_quants(nmax, data, quant_weights):scale_is = group_max / (0.1 * _is + nmax)iscale_is = get_reciprocal(scale_is)- tmp_L = torch.round(iscale_is * data).clip(max=nmax)+ tmp_L = torch.round(iscale_is * data + v).clip(max=nmax)diffs = data - scale_is * tmp_Lmse = torch.sum(quant_weights * diffs * diffs, dim=-1)@@ -221,7 +280,7 @@ def make_qp_quants(nmax, data, quant_weights):best_mse[replace_idx] = mse[replace_idx]iscale[replace_idx] = iscale_is[replace_idx]- L = torch.round(iscale * data).clip(max=nmax)+ L = torch.round(iscale * data + v).clip(max=nmax)sumlx = torch.sum(quant_weights * data * L, dim=-1)suml2 = torch.sum(quant_weights * L * L, dim=-1)#@@ -285,7 +344,7 @@ def quant_tensor_asym_dq(Quantized and de-quantized tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2**bits - 1+ maxq = int(2.0**bits) - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -304,7 +363,7 @@ def quant_tensor_asym_dq(wmin = -wmin # pylint: disable=E1130wmin = wmin.view(-1, super_group_size)- ##conduct double quant+ # Conduct double quantscale, d_scale = double_quant_tensor(scale, super_bits)wmin, d_wmin = double_quant_tensor(wmin, super_bits)@@ -356,7 +415,7 @@ def _imatrix_handle_zero(imatrix: Union[torch.Tensor, float], weight: torch.Tens@torch.inference_mode()-def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrix=None, split_num=1):+def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrix=None, split_num=1, v=0):super_bits = 4 if bits == 2 else 6super_group_size = 16 if bits == 2 else 8@@ -383,6 +442,7 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatriuse_mad=params["use_mad"],weights=quant_weights,split_num=split_num,+ v=v,)scale = scale.to(scale_dtype)scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)@@ -426,10 +486,11 @@ def search_gguf_scale_min_asym(tensor, bits=4, scale_dtype=torch.float16, imatrinstep=params["nstep"],use_mad=params["use_mad"],weights=quant_weights,+ v=v,)scale = scale.to(scale_dtype)scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)- nmax = 2**super_bits - 1+ nmax = int(2.0**super_bits) - 1scale = scale.reshape(-1, super_group_size)wmin = wmin_0.reshape(-1, super_group_size)sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)@@ -470,7 +531,6 @@ def quant_tensor_gguf_asym_dq(Args:tensor (torch.Tensor): Input tensor to quantize.bits (int): Number of bits for quantization.- v (float): Perturbation added before rounding.scale_dtype (torch.dtype): Data type for quantized scale.imatrix (torch.Tensor, optional): Importance matrix for weighted quantization.@@ -480,7 +540,7 @@ def quant_tensor_gguf_asym_dq(if bits not in [2, 4, 5]:raise ValueError(f"bits={bits} not supported by rtn_int_asym_dq")orig_dtype = tensor.dtype- maxq = 2**bits - 1+ maxq = int(2.0**bits) - 1group_size = 16 if bits == 2 else 32split_num = 1@@ -497,7 +557,12 @@ def quant_tensor_gguf_asym_dq(d_wmin = prev_d_wmin.detach()if scale is None:scale, wmin, d_scale, d_wmin = search_gguf_scale_min_asym(- tensor, bits, scale_dtype, imatrix, split_num=split_num+ tensor,+ bits,+ scale_dtype,+ imatrix,+ split_num=split_num,+ v=v,)scale = scale.clone()wmin = wmin.clone()@@ -515,13 +580,15 @@ def quant_tensor_gguf_asym_dq(# TODO consolidate iterative_wls_quant_search_chunk and non-chunkdef iterative_wls_quant_search_chunk(- data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, split_num=1+ data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, split_num=1, v=0):dtype = torch.float32data = data.to(dtype)- maxq = 2**bits - 1+ maxq = int(2.0**bits) - 1minq = 0weights = 1.0 if weights is None else weights.to(dtype)+ # A 0-dim tensor (scalar tensor) cannot be sliced; treat it like a Python scalar.+ v_is_tensor = isinstance(v, torch.Tensor) and v.dim() > 0results_scale = []results_rmin = []@@ -532,6 +599,7 @@ def iterative_wls_quant_search_chunk(end = min(start + chunk_size, data.shape[0])chunk = data[start:end]chunk_weights = weights if isinstance(weights, float) else weights[start:end]+ v_chunk = v[start:end] if v_is_tensor else v# Pre-allocate reusable buffers to avoid new allocationstmp = torch.empty_like(chunk)@@ -546,8 +614,10 @@ def iterative_wls_quant_search_chunk(scale = (rmax - rmin) / (maxq - minq)iscale = get_reciprocal(scale)- # tmp = (chunk - rmin) * iscale+ # tmp = (chunk - rmin) * iscale + vtmp.copy_(chunk).sub_(rmin).mul_(iscale)+ if v_is_tensor or v != 0:+ tmp.add_(v_chunk)# quant_data = round(tmp).clamp_()torch.round(tmp, out=quant_data)@@ -568,8 +638,10 @@ def iterative_wls_quant_search_chunk(scale_new = (rmax - rmin) / factoriscale_new = get_reciprocal(scale_new)- # tmp = (chunk - rmin) * iscale_new+ # tmp = (chunk - rmin) * iscale_new + vtmp.copy_(chunk).sub_(rmin).mul_(iscale_new)+ if v_is_tensor or v != 0:+ tmp.add_(v_chunk)torch.round(tmp, out=quant_data)quant_data.clamp_(minq, maxq)@@ -593,8 +665,10 @@ def iterative_wls_quant_search_chunk(reverse_this_scale = get_reciprocal(this_scale)- # tmp = (chunk - this_min) * reverse_this_scale+ # tmp = (chunk - this_min) * reverse_this_scale + vtmp.copy_(chunk).sub_(this_min).mul_(reverse_this_scale)+ if v_is_tensor or v != 0:+ tmp.add_(v_chunk)torch.round(tmp, out=quant_data)quant_data.clamp_(minq, maxq)@@ -626,7 +700,7 @@ def iterative_wls_quant_search_chunk(def iterative_wls_quant_search(- data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, split_num=1+ data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None, split_num=1, v=0):"""Adapted from Llamacpp. Performs iterative weighted least squares quantization search.@@ -638,6 +712,8 @@ def iterative_wls_quant_search(nstep (int): Number of search steps.use_mad (bool): Whether to use mean absolute deviation instead of squared error.weights (torch.Tensor): Weight matrix for each element.+ v: Optional rounding perturbation (scalar or tensor with the same+ shape as ``data``) for AutoRound-style tuning.Returns:Tuple: (Optimal scale tensor, optimal minimum value tensor)@@ -654,19 +730,20 @@ def iterative_wls_quant_search(use_mad=use_mad,weights=weights,split_num=split_num,+ v=v,)@torch.inference_mode()-def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num):+def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num, v=0):if imatrix is None or (imatrix is not None and torch.sum(imatrix) == 0):if bits == 3:# Note: make_q3_quants does not support split_num/chunking;# 3-bit quantization is performed in a single chunk.- scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True)+ scale, int_w = make_q3_quants(tensor, bits=bits, do_rmse=True, v=v)# scale, int_w = make_qx_quants(tensor, bits=bits, rmse_type=1, qw=None)elif bits == 6:- scale, int_w = make_qx_quants_chunk(tensor, bits=bits, rmse_type=1, qw=None, split_num=split_num)+ scale, int_w = make_qx_quants_chunk(tensor, bits=bits, rmse_type=1, qw=None, split_num=split_num, v=v)else:imatrix = imatrix.to(tensor.device)weights = imatrix.reshape(1, -1)@@ -675,7 +752,7 @@ def search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num):quant_weights = _imatrix_handle_zero(quant_weights, tensor, bits)- scale, int_w = make_qx_quants_chunk(tensor, bits=bits, rmse_type=1, qw=quant_weights, split_num=split_num)+ scale, int_w = make_qx_quants_chunk(tensor, bits=bits, rmse_type=1, qw=quant_weights, split_num=split_num, v=v)if split_num > 1:clear_memory(device_list=[tensor.device])return scale@@ -701,7 +778,6 @@ def quant_tensor_gguf_sym_dq(Args:tensor: Tensor containing the tensor to be quantizedbits: Number of bits for quantization (e.g., 2, 3, 4, 8)- v: Rounding value perturbationmin_scale: Minimum scale coefficient for tensormax_scale: Maximum scale coefficient for tensortensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.@@ -718,7 +794,7 @@ def quant_tensor_gguf_sym_dq(if bits not in [3, 6]:raise KeyError(f"bits={bits} is not supported by gguf_int_sym_dq, please check.")- maxq = 2 ** (bits - 1)+ maxq = int(2.0 ** (bits - 1))group_size = 16tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)orig_dtype = tensor.dtype@@ -737,7 +813,7 @@ def quant_tensor_gguf_sym_dq(scale = prev_scale.detach()d_scale = prev_d_scale.detach()if scale is None or d_scale is None:- scale = search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num=split_num)+ scale = search_gguf_scale_min_sym(tensor, bits, imatrix, scale_dtype, split_num=split_num, v=v)scale = scale.to(scale_dtype)scale = torch.where(torch.abs(scale) < 1e-30, torch.zeros_like(scale), scale)scale, d_scale = double_quant_tensor_sym_rtn(scale, super_bits)diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex 897dbb17a..9c7e47243 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -23,7 +23,7 @@ from auto_round.utils import get_reciprocaldef search_scales(data: torch.Tensor, bits: int, qw: Union[None, torch.Tensor, float] = None) -> torch.Tensor:# Maximum absolute value for symmetric quantization- nmax = 1 << (bits - 1) # equivalent to pow(2, bits-1)+ nmax = int(2.0 ** (bits - 1))# Find per-group max along the last dimensionimax = torch.abs(data).argmax(dim=-1, keepdim=True)@@ -103,7 +103,7 @@ def quant_tensor_opt_rtn_sym(tensor, bits=4, group_size=-1, v=0, q_scale_thresh=from auto_round.data_type.gguf import _imatrix_handle_zerotensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** (bits - 1)+ maxq = int(2.0 ** (bits - 1))if imatrix is None:imatrix = 1.0else:@@ -146,7 +146,7 @@ def quant_tensor_rtn_sym(Quantized and de-quantized tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** (bits - 1)+ maxq = int(2.0 ** (bits - 1))wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -196,7 +196,8 @@ def quant_tensor_sym("""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2 ** (bits - 1)+ maxq = int(2.0 ** (bits - 1))+if init_scale is not None:scale = init_scale * max_scale.unsqueeze(dim=-1)scale = scale.to(scale_dtype)@@ -259,7 +260,7 @@ def quant_tensor_asym(Quantized and de-quantized tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2**bits - 1+ maxq = int(2.0**bits) - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -316,7 +317,7 @@ def quant_tensor_sym_gptq(Quantized and de-quantized tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2**bits - 1+ maxq = int(2.0**bits) - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)@@ -379,7 +380,7 @@ def quant_tensor_asym_wo_round(Quantized and de-quantize tensor, scale, zero-point"""tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)- maxq = 2**bits - 1+ maxq = int(2.0**bits) - 1if tensor_min is None or tensor_max is None:wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex 92b13d045..b1b60dc6b 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -130,21 +130,29 @@ def torch_roundf(n):return torch.sign(n) * b-def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1):+def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1, v=0):"""Extreme VRAM-optimized version of quantization.- Processes data in chunks along the batch dimension (dim=0) to reduce peak memory usage.- Uses inplace operations to avoid unnecessary tensor copies.- Reuses buffers for temporary calculations wherever possible.++ Args:+ v: Optional rounding perturbation. Either a scalar or a tensor with the+ same shape as ``data``; added before each ``round`` to support+ AutoRound-style tuning."""nmax = 2 ** (bits - 1)scales_list = []L_list = []chunk_size = (data.shape[0] + split_num - 1) // split_num+ # A 0-dim tensor (scalar tensor) cannot be sliced; treat it like a Python scalar.+ v_is_tensor = isinstance(v, torch.Tensor) and v.dim() > 0for start in range(0, data.shape[0], chunk_size):end = min(start + chunk_size, data.shape[0])chunk = data[start:end] # Slice a batch chunk to reduce memory footprint+ v_chunk = v[start:end] if v_is_tensor else v# Compute absolute values inplace to avoid extra tensor allocationchunk_abs = chunk.abs()@@ -156,7 +164,7 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1):iscales = -nmax * get_reciprocal(group_max)# L buffer stores quantized values, modified inplace to save memory- L = (chunk * iscales).round_().clamp_(-nmax, nmax - 1)+ L = (chunk * iscales + v_chunk).round_().clamp_(-nmax, nmax - 1)# Simple case: rmse_type == 0if rmse_type == 0:@@ -204,7 +212,7 @@ def make_qx_quants_chunk(data, bits, rmse_type=0, qw=None, split_num=1):continueiscales_tmp = -(nmax + -0.1 * _is) / group_max# Use a temporary L buffer to avoid creating new large tensor- L_tmp = (chunk * iscales_tmp).round_().clamp_(-nmax, nmax - 1)+ L_tmp = (chunk * iscales_tmp + v_chunk).round_().clamp_(-nmax, nmax - 1)sumlx_tmp = (w * chunk * L_tmp).sum(dim=-1)suml2_tmp = (w * L_tmp * L_tmp).sum(dim=-1)# Determine which elements should be replaced@@ -283,7 +291,7 @@ def make_qx_quants(data, bits, rmse_type=0, qw=None):return scales, L-def make_q3_quants(data, bits, do_rmse=False):+def make_q3_quants(data, bits, do_rmse=False, v=0):# Maximum absolute integer value for symmetric quantizationnmax = 1 << (bits - 1) # equivalent to pow(2, bits-1)@@ -299,7 +307,7 @@ def make_q3_quants(data, bits, do_rmse=False):if do_rmse:# Initial quantization L (in-place round and clamp)L = torch.empty_like(data)- torch.round(iscale * data, out=L)+ torch.round(iscale * data + v, out=L)L.clamp_(-nmax, nmax - 1)# Weight for RMSE = x^2 (in-place)@@ -349,7 +357,7 @@ def make_q3_quants(data, bits, do_rmse=False):# Fast path: quantize without RMSE (in-place round, clamp, shift)L = torch.empty_like(data)- torch.round(iscale * data, out=L)+ torch.round(iscale * data + v, out=L)L.clamp_(-nmax, nmax - 1)L.add_(nmax)