PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
GGUF support step2: add naive Q2_KS and Q4_KS (#448)
Heng Guo12539ef5aab0
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 1bfca8b89..abd7750aa 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -162,6 +162,8 @@ class AutoRound(object):enable_norm_bias_tuning: bool = False,enable_torch_compile: bool = False,device_map: Union[str, dict] = None,+ super_bits: int = None,+ super_group_size: int = None,**kwargs,):self.quantized = False@@ -223,6 +225,9 @@ class AutoRound(object):self.batch_dim = Noneself.infer_bs_coeff = 1+ self.super_bits = super_bits+ self.super_group_size = super_group_size+torch.set_printoptions(precision=3, sci_mode=True)self.check_configs()if self.act_bits <= 8 and self.amp_dtype == torch.float16:@@ -523,7 +528,7 @@ class AutoRound(object):"""layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)keys = ["data_type", "bits", "group_size", "sym", "scale_dtype", "act_bits", "act_group_size", "act_sym",- "act_dynamic", "act_data_type"]+ "act_dynamic", "act_data_type", "super_bits", "super_group_size"]for n, m in self.model.named_modules():##delete keys to avoid conflict with the previous tuningfor key in keys:@@ -1662,6 +1667,8 @@ class AutoRoundOPT(AutoRound):enable_torch_compile: bool = False,device_map: Union[str, dict] = None,optimizer="AdamW",+ super_bits: int = None,+ super_group_size: int = None,**kwargs,):super(AutoRoundOPT, self).__init__(@@ -1702,6 +1709,8 @@ class AutoRoundOPT(AutoRound):enable_norm_bias_tuning=enable_norm_bias_tuning,enable_torch_compile=enable_torch_compile,device_map=device_map,+ super_bits=super_bits,+ super_group_size=super_group_size,**kwargs,)@@ -1836,6 +1845,8 @@ class AutoRoundAdam(AutoRoundOPT):enable_torch_compile: bool = False,device_map: Union[str, dict] = None,optimizer="AdamW",+ super_bits: int = None,+ super_group_size: int = None,**kwargs,):super(AutoRoundAdam, self).__init__(@@ -1877,5 +1888,7 @@ class AutoRoundAdam(AutoRoundOPT):enable_torch_compile=enable_torch_compile,device_map=device_map,optimizer=optimizer,+ super_bits=super_bits,+ super_group_size=super_group_size,**kwargs,)diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex 26e5a4d7a..899abf17a 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -65,7 +65,7 @@ def quant_tensor_sym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scal## the values should be positivedef double_quant_tensor(tensor, bits, q_scale_thresh):- maxq = 2 ** bits+ maxq = 2 ** bits - 1wmax = torch.clamp(tensor.max(-1)[0], min=0)scale = torch.clamp(wmax / maxq, q_scale_thresh)scale = scale.view(-1, 1)@@ -75,7 +75,7 @@ def double_quant_tensor(tensor, bits, q_scale_thresh):@register_dtype("int_asym_dq")def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,- tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, super_group_size=32, super_bits=6,+ tensor_min=None, tensor_max=None, q_scale_thresh=1e-5, super_group_size=8, super_bits=6,**kwargs):"""Quantize and de-quantize tensor asymmetrically.@@ -128,7 +128,7 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_qdq_result = (scale * q - wmin_m).to(tensor.dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)zp = round_ste(wmin_m / scale) # remove this later- return qdq_result, scale, zp+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}@register_dtype("int_asym")diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pynew file mode 100644index 000000000..2d7b1f163--- /dev/null+++ b/auto_round/export/export_to_gguf/config.py@@ -0,0 +1,41 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++GGUF_CONFIG = {}++GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": False, "data_type": "int"}++GGUF_CONFIG["gguf:q4_1"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": True, "data_type": "int"}++GGUF_CONFIG["gguf:q4_k_s"] = {+ "bits": 4,+ "act_bits": 16,+ "super_group_size": 8,+ "super_bits": 6,+ "group_size": 32,+ "asym": True,+ "data_type": "int_asym_dq"+}++GGUF_CONFIG["gguf:q2_k_s"] = {+ "bits": 2,+ "act_bits": 16,+ "super_group_size": 16,+ "super_bits": 4,+ "group_size": 16,+ "asym": True,+ "data_type": "int_asym_dq"+}++GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "asym": False, "data_type": "int"}diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 9024947de..9d3c4b852 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -1253,16 +1253,16 @@ class Model(OriModel):data_qtype = gguf.GGMLQuantizationType.F16elif self.ftype == gguf.LlamaFileType.MOSTLY_BF16:data_qtype = gguf.GGMLQuantizationType.BF16- # elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:- # data_qtype = gguf.GGMLQuantizationType.Q8_0- # elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ1_0:- # data_qtype = gguf.GGMLQuantizationType.TQ1_0- # elif self.ftype == gguf.LlamaFileType.MOSTLY_TQ2_0:- # data_qtype = gguf.GGMLQuantizationType.TQ2_0+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q8_0:+ data_qtype = gguf.GGMLQuantizationType.Q8_0elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_0:data_qtype = gguf.GGMLQuantizationType.Q4_0elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:data_qtype = gguf.GGMLQuantizationType.Q4_1+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:+ data_qtype = gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:+ data_qtype = gguf.GGMLQuantizationType.Q2_Kelse:raise ValueError(f"Unknown file type: {self.ftype.name}")@@ -1285,11 +1285,23 @@ class Model(OriModel):scale = module.scaleif isinstance(scale, torch.Tensor):scale = scale.numpy()- zp = module.zp+ zp = module.zp if hasattr(module, "zp") else Noneif isinstance(zp, torch.Tensor):zp = zp.numpy()- data = ggml_quant(data, data_qtype.name.lower(), scale,- zp)+ if data_qtype.name.lower().endswith("_k"):+ d_scale = module.w_d_scale.numpy()+ d_wmin_m = module.w_d_wmin_m.numpy()+ wmin_m = module.w_wmin_m.numpy()+ data = ggml_quant(+ data,+ data_qtype.name.lower(),+ scale,+ zp,+ wmin_m=wmin_m,+ d_scale=d_scale,+ d_wmin_m=d_wmin_m)+ else:+ data = ggml_quant(data, data_qtype.name.lower(), scale, zp)else:data_qtype = gguf.GGMLQuantizationType.F32else:diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 211e1f00e..e5af13201 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -19,6 +19,7 @@ import torchfrom .convert import Modelfrom auto_round.utils import logger, LazyImportfrom pathlib import Path+import timegguf = LazyImport("gguf")@@ -30,12 +31,15 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {"q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,"q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,"q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,- "q4_k": gguf.LlamaFileType.MOSTLY_Q4_K_S,+ "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,+ "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,+ "q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,"auto": gguf.LlamaFileType.GUESSED,}def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):"""Export the model to gguf format."""+ st = time.time()model = kwargs["model"]tokenizer = kwargs.get("tokenizer", None)@@ -77,7 +81,8 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):dry_run=False,small_first_shard=False)model_instance.write()- logger.info(f"Model successfully exported to {model_instance.fname_out}")+ rt = time.time() - st+ logger.info(f"Model successfully exported to {model_instance.fname_out}, running time={rt}")shutil.rmtree(tmp_work_dir, ignore_errors=True)diff --git a/auto_round/export/export_to_gguf/quant.py b/auto_round/export/export_to_gguf/quant.pyindex aa12b6ee3..1b0be3a80 100644--- a/auto_round/export/export_to_gguf/quant.py+++ b/auto_round/export/export_to_gguf/quant.py@@ -13,38 +13,80 @@# limitations under the License.import numpy as np+from tqdm import tqdm+from concurrent.futures import ProcessPoolExecutorQK_K = 256+K_SCALE_SIZE = 12GGML_QUANT_SIZES = {"bf16": (1, 2),"q4_0": (32, 2 + 16),"q4_1": (32, 2 + 2 + 16),- "q4_k": (256, 2 + 2 + QK_K // 2 + 12),+ "q4_k": (256, 2 + 2 + QK_K//2 + 12),+ "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),+ "q8_0": (32, 2 + 32)}GGML_QUANT_BLOCK = {}+def register_block(name):+def register(cls):GGML_QUANT_BLOCK[name] = clsreturn cls+return register-def ggml_quant(data: np.array, ggml_type, scale = None, zp = None):+def ggml_quant(data: np.array, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, worker=16):block_size, type_size = GGML_QUANT_SIZES[ggml_type]data = data.astype(np.float32, copy=False)shape = data.shapen_blocks = data.size // block_sizeblocks = data.reshape((n_blocks, block_size))-- new_data = GGML_QUANT_BLOCK[ggml_type](blocks, scale, zp)++ if worker > 0:+ n_groups = (data.shape[0] // worker) or 1+ blocks = np.array_split(blocks, n_groups, axis=0)+ scale = np.array_split(scale, n_groups, axis=0) if scale is not None else [None] * n_groups+ zp = np.array_split(zp, n_groups, axis=0) if zp is not None else [None] * n_groups+ wmin_m = np.array_split(wmin_m, n_groups, axis=0) if wmin_m is not None else [None] * n_groups+ d_scale = np.array_split(d_scale, n_groups, axis=0) if d_scale is not None else [None] * n_groups+ d_wmin_m = np.array_split(d_wmin_m, n_groups, axis=0) if d_wmin_m is not None else [None] * n_groups++ quant_func = GGML_QUANT_BLOCK[ggml_type]+ if ggml_type.endswith("_k"):+ with ProcessPoolExecutor(worker) as executor:+ result = executor.map(quant_func, blocks, scale, zp, wmin_m, d_scale, d_wmin_m)+ new_data = np.array(list(result), dtype=np.uint8)+ else:+ with ProcessPoolExecutor(n_groups) as executor:+ result = executor.map(quant_func, blocks, scale, zp)+ new_data = np.array(list(result), dtype=np.uint8)+ else:+ quant_func = GGML_QUANT_BLOCK[ggml_type]+ if ggml_type.endswith("_k"):+ new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)+ else:+ new_data = quant_func(blocks, scale, zp)++ assert new_data.dtype == np.uint8+ assert new_data.shape[-1] == type_sizenew_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)return new_data++def np_roundf(n: np.ndarray) -> np.ndarray:+ a = abs(n)+ floored = np.floor(a)+ b = floored + np.floor(2 * (a-floored))+ return np.sign(n) * b++@register_block("bf16")-def bf16_quant_block(blocks: np.array, scale = None, zp = None):+def bf16_quant_block(blocks: np.array, scale=None, zp=None):n = blocks.view(np.uint32)# force nan to quietn = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | np.uint32(64 << 16), n)@@ -52,10 +94,11 @@ def bf16_quant_block(blocks: np.array, scale = None, zp = None):n = (np.uint64(n) + (0x7fff + ((n >> 16) & 1))) >> 16return n.astype(np.uint16).view(np.uint8)+@register_block("q4_0")-def q4_0_quant_block(blocks: np.array, scale = None, zp = None):+def q4_0_quant_block(blocks: np.array, scale=None, zp=None):if scale is not None:- d = scale.reshape((-1,1))+ d = scale.reshape((-1, 1))else:imax = abs(blocks).argmax(axis=-1, keepdims=True)max = np.take_along_axis(blocks, imax, axis=-1)@@ -63,9 +106,8 @@ def q4_0_quant_block(blocks: np.array, scale = None, zp = None):with np.errstate(divide="ignore"):id = np.where(d == 0, 0, 1 / d)- qs = np.trunc(- (np.float64(blocks) * np.float64(id)) + np.float64(8.5),- dtype=np.float32).astype(np.uint8).clip(0, 15)+ qs = np.trunc((np.float64(blocks) * np.float64(id)) + np.float64(8.5),+ dtype=np.float32).astype(np.uint8).clip(0, 15)n_blocks = blocks.shape[0]block_size = GGML_QUANT_SIZES["q4_0"][0]@@ -78,18 +120,18 @@ def q4_0_quant_block(blocks: np.array, scale = None, zp = None):@register_block("q4_1")-def q4_1_quant_block(blocks: np.array, scale = None, zp = None):+def q4_1_quant_block(blocks: np.array, scale=None, zp=None):if scale is not None:- d = scale.reshape((-1,1))- min = zp.reshape((-1,1)) * d * -1+ d = scale.reshape((-1, 1))+ min = zp.reshape((-1, 1)) * d * -1else:max = blocks.max(axis=-1, keepdims=True)min = blocks.min(axis=-1, keepdims=True)- d = (max - min) / 15+ d = (max-min) / 15with np.errstate(divide="ignore"):id = np.where(d == 0, 0, 1 / d)- qs = np.trunc((blocks - min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 15)+ qs = np.trunc((blocks-min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 15)n_blocks = blocks.shape[0]block_size = GGML_QUANT_SIZES["q4_1"][0]@@ -99,3 +141,239 @@ def q4_1_quant_block(blocks: np.array, scale = None, zp = None):d = d.astype(np.float16).view(np.uint8)m = min.astype(np.float16).view(np.uint8)return np.concatenate([d, m, qs], axis=-1)+++@register_block("q8_0")+def q8_0_quant_block(blocks: np.array, scale=None, zp=None) -> np.ndarray:+ if scale is not None:+ d = scale.reshape((-1, 1))+ else:+ d = abs(blocks).max(axis=1, keepdims=True) / 127+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ qs = np_roundf(blocks * id)++ # (n_blocks, 2)+ d = d.astype(np.float16).view(np.uint8)+ # (n_blocks, block_size)+ qs = qs.astype(np.int8).view(np.uint8)++ return np.concatenate([d, qs], axis=1)+++def make_qkx2_quants(data, weight, nmax, group_size, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):+ group_min = np.min(data)+ group_max = np.max(data)++ sum_w = np.sum(weight)+ sum_x = np.sum(weight * data)++ if group_min > 0:+ group_min = 0+ if group_min == group_max:+ L = np.zeros(group_size, dtype=np.uint8)+ the_min = -group_min+ return 0.0, L, the_min++ iscale = nmax / (group_max-group_min)+ scale = 1 / iscale+ L = np.zeros(group_size, dtype=np.uint8)++ l_values = np.round(iscale * (data-group_max))+ L = np.clip(l_values, 0, nmax).astype(np.uint8)+ diffs = scale*L + group_min - data+ diffs = np.abs(diffs) if use_mad else diffs**2+ best_mad = np.sum(weight * (diffs))++ if nstep < 1:+ the_min = -group_min+ return scale, L, the_min++ Laux = []+ for step in range(nstep):+ iscale = (rmin + rdelta*step + nmax) / (group_max-group_min)+ l_values = np.round(iscale * (data-group_min))+ Laux = np.clip(l_values, 0, nmax).astype(np.uint8)++ sum_l = np.sum(weight * Laux)+ sum_l2 = np.sum(weight * Laux**2)+ sum_xl = np.sum(weight * Laux * data)++ D = sum_w*sum_l2 - sum_l*sum_l+ if D > 0:+ this_scale = (sum_w*sum_xl - sum_x*sum_l) / D+ this_min = (sum_l2*sum_x - sum_l*sum_xl) / D+ if this_min > 0:+ this_min = 0+ this_scale = sum_xl / sum_l2++ diffs = this_scale*Laux + this_min - data+ mad = np.sum(weight * diffs**2)++ if mad < best_mad:+ L = Laux.copy()+ best_mad = mad+ scale = this_scale+ group_min = this_min++ the_min = -group_min+ return scale, L, the_min+++@register_block("q2_k")+def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ output_scale = np.empty((nb, 16), dtype=np.uint8)+ output_d = np.empty(nb, dtype=np.float32)+ output_dmin = np.empty(nb, dtype=np.float32)+ output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)++ blocks = blocks.reshape((nb, QK_K // 16, 16))+ weight = np.abs(blocks)+ scales = np.empty(QK_K // 16, dtype=np.float32)+ mins = np.empty((QK_K // 16), dtype=np.float32)++ if scale is not None:+ scale = scale.reshape((-1, QK_K // 16))+ wmin_m = wmin_m.reshape((-1, QK_K // 16))+ output_d = d_scale.astype(np.float32)+ output_dmin = d_wmin_m.astype(np.float32)+ inv_scales = np.where(d_scale == 0, 0, 1 / output_d)+ inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)++ # for i in tqdm(range(nb), desc="packing layer"):+ for i in range(nb):+ all_L = np.empty(blocks[i].shape, dtype=np.uint8)++ if scale is not None:+ scales = scale[i]+ mins = wmin_m[i]+ inv_scale = inv_scales[i]+ inv_min = inv_mins[i]+ max_scale = max(scales)+ max_min = max(mins)+ all_L = np.round((blocks[i] + mins.reshape(-1, 1)) / scales.reshape(-1, 1)).astype(np.uint8)+ else:+ for j in range(QK_K // 16):+ tmp_scale, tmp_l, the_min = make_qkx2_quants(+ blocks[i][j], weight[i][j], nmax=3, group_size=16, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)+ all_L[j] = tmp_l+ scales[j] = tmp_scale+ mins[j] = the_min++ max_scale = max(scales)+ max_min = max(mins)+ inv_scale = 15. / max_scale+ inv_min = 15. / max_min++ if max_scale > 0:+ output_scale[i] = np.round(inv_scale * scales).astype(np.uint8)+ output_d[i] = max_scale / 15.+ else:+ output_scale[i] = np.zeros_like(scales).astype(np.uint8)+ output_d[i] = 0.+ if max_min > 0:+ output_scale[i] |= (np.round(inv_min * mins).astype(np.uint8) << 4)+ output_dmin[i] = max_min / 15.+ else:+ output_dmin[i] = 0.++ d_tmp = output_d[i] * (output_scale[i] & 0xF)+ dm_tmp = output_dmin[i] * (output_scale[i] >> 4)+ for j in range(QK_K // 16):+ if d_tmp[j] == 0.:+ continue+ else:+ all_L[j] = np.round((blocks[i][j] + dm_tmp[j]) / d_tmp[j]).astype(np.uint8)+ all_L = np.clip(all_L, 0, 3)+ output_qs[i] = all_L[::4] | (all_L[1::4] << 2) | (all_L[2::4] << 4) | (all_L[3::4] << 6)++ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_qs = output_qs.reshape((nb, QK_K // 4))++ # [scale, qs, d, dmin]+ return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)+++@register_block("q4_k")+def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)+ output_d = np.empty(nb, dtype=np.float32)+ output_dmin = np.empty(nb, dtype=np.float32)+ output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)++ blocks = blocks.reshape((nb, QK_K // 32, 32))+ sum_x2 = np.sum(np.power(blocks, 2), axis=-1)+ av_x = np.sqrt(sum_x2 / 32)+ weight = blocks + av_x.reshape((*av_x.shape, 1))+ scales = np.empty(QK_K // 32, dtype=np.float32)+ mins = np.empty(QK_K // 32, dtype=np.float32)++ if scale is not None:+ scale = scale.reshape(-1, QK_K // 32)+ wmin_m = wmin_m.reshape(-1, QK_K // 32)+ output_d = d_scale.astype(np.float32)+ output_dmin = d_wmin_m.astype(np.float32)+ inv_scales = np.where(d_scale == 0, 0, 1 / output_d)+ inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)++ # for i in tqdm(range(nb), desc="packing layer"):+ for i in range(nb):+ all_L = np.empty(blocks[i].shape, dtype=np.uint8)++ if scale is not None:+ scales = scale[i]+ mins = wmin_m[i]+ inv_scale = inv_scales[i]+ inv_min = inv_mins[i]+ max_scale = max(scales)+ max_min = max(mins)+ all_L = np.round((blocks[i] + mins.reshape(-1, 1)) / scales.reshape(-1, 1)).astype(np.uint8)+ else:+ for j in range(QK_K // 32):+ tmp_scale, tmp_l, the_min = make_qkx2_quants(+ blocks[i][j], weight[i][j], nmax=15, group_size=32, rmin=-1, rdelta=0.1, nstep=20)+ all_L[j] = tmp_l+ scales[j] = tmp_scale+ mins[j] = the_min++ max_scale = max(scales)+ max_min = max(mins)+ inv_scale = 63. / max_scale if max_scale > 0 else 0.+ inv_min = 63. / max_min if max_min > 0 else 0.++ ls = np.round(inv_scale * scales).astype(np.uint8)+ lm = np.round(inv_min * mins).astype(np.uint8)++ output_scale[i][:4] = ls[:4]+ output_scale[i][4:8] = lm[:4]++ output_scale[i][8:] = (ls[4:] & 0xF) | ((lm[4:] & 0xF) << 4)+ output_scale[i][:4] |= ((ls[4:] >> 4) << 6)+ output_scale[i][4:8] |= ((lm[4:] >> 4) << 6)++ if d_scale is None:+ output_d[i] = max_scale / 63+ if d_wmin_m is None:+ output_dmin[i] = max_min / 63++ d_tmp = output_d[i] * ls+ dm_tmp = output_dmin[i] * lm++ for j in range(d_tmp.size):+ if d_tmp[j] == 0.:+ continue+ else:+ all_L[j] = np.round((blocks[i][j] + dm_tmp[j]) / d_tmp[j]).astype(np.uint8)++ all_L = np.clip(all_L, 0, 15)+ output_qs[i] = all_L[::2] | (all_L[1::2] << 4)++ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_qs = output_qs.reshape(nb, QK_K // 2)++ # [d, dmin, scale, qs]+ return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 9765fff0b..2efe4d499 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -186,6 +186,12 @@ class BasicArgumentParser(argparse.ArgumentParser):self.add_argument("--device_map", default=None, type=str, help="device_map for block in tuning phase")+ self.add_argument(+ "--super_group_size", default=None, type=int, help="the number of super group size when use double quant.")++ self.add_argument(+ "--super_bits", default=None, type=int, help="number of scale and mins quant bits for double quant.")+class EvalArgumentParser(argparse.ArgumentParser):@@ -304,24 +310,10 @@ def setup_eval_parser():def _gguf_args_check(args):from auto_round.utils import logger-- _GGUF_CONFIG = {- "gguf:q4_0": {- "bits": 4,- "act_bits": 16,- "group_size": 32,- "asym": False,- },- "gguf:q4_1": {- "bits": 4,- "act_bits": 16,- "group_size": 32,- "asym": True,- }- }+ from auto_round.export.export_to_gguf.config import GGUF_CONFIGformats = args.format.lower().replace(' ', '').split(",")- for format in _GGUF_CONFIG:+ for format in GGUF_CONFIG:if format in formats:from pathlib import Pathfrom auto_round.export.export_to_gguf.convert import Model@@ -333,8 +325,17 @@ def _gguf_args_check(args):logger.error(f"Model {model_architecture} is not supported to export GGUF format")sys.exit(1)+ if format.endswith("_k") and ("hidden_size" in hparams and hparams["hidden_size"] % 256 !=0):+ model_name = args.model.split('/')+ model_name = model_name[-1] if model_name[-1] else model_name[-2]+ hidden_size = hparams["hidden_size"]+ logger.error(+ f"Currently only support pure mode for format: {format}. "+ f"{model_name} is not supported, cause hidden_size({hidden_size}) % 256 !=0")+ sys.exit(-1)+unsupport_list, reset_list = [], []- gguf_config = _GGUF_CONFIG[format]+ gguf_config = GGUF_CONFIG[format]for k, v in gguf_config.items():if getattr(args, k) != v:unsupport_list.append(f"{k}={getattr(args, k)}")@@ -363,14 +364,17 @@ def tune(args):from auto_round import AutoRoundConfigfrom auto_round.utils import detect_device, get_library_version, detect_device_countfrom auto_round.utils import logger+ from auto_round.export.export_to_gguf.config import GGUF_CONFIGtasks = args.tasksif args.format is None:args.format = "auto_round"supported_formats = ["auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:auto_awq", "auto_gptq:marlin",- "gguf:q4_0", "gguf:q4_1", "itrex", "itrex_xpu", "fake"+ "itrex", "itrex_xpu", "fake"]+ supported_formats.extend(list(GGUF_CONFIG.keys()))+formats = args.format.lower().replace(' ', '').split(",")for format in formats:if format not in supported_formats:@@ -564,7 +568,10 @@ def tune(args):enable_torch_compile=enable_torch_compile,act_data_type=args.act_data_type,act_dynamic=not args.disable_act_dynamic,- device_map=args.device_map)+ device_map=args.device_map,+ super_group_size=args.super_group_size,+ super_bits=args.super_bits,+ )model, _ = autoround.quantize()model_name = args.model.rstrip("/")if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:@@ -582,17 +589,18 @@ def tune(args):format_list = args.format.replace(' ', '').split(',')inplace = False if len(format_list) > 1 else Trueeval_folder = None++ eval_gguf_model = Falsefor format_ in format_list:save_format_ = format_.replace(":", "-")save_format_ = save_format_.replace("_", "-")save_folder = f'{export_dir}-{save_format_}'autoround.save_quantized(save_folder, format=format_, inplace=inplace)if 'gguf' in format_:- gguf_format = format_.upper().split(":")[-1]- if not any([file.endswith(f"{gguf_format}.gguf") for file in os.listdir(save_folder)]):+ if not any([file.endswith(".gguf") for file in os.listdir(save_folder)]):logger.error(f"fail to export {format_}")- else:- eval_folder = save_folder+ eval_gguf_model = True+ eval_folder = save_folderlm_eval_version = get_library_version("lm-eval")@@ -604,7 +612,11 @@ def tune(args):logger.info(f"Using lm-eval version {lm_eval_version}")- if args.act_bits <= 8:+ if args.act_bits <= 8 or eval_gguf_model:+ if eval_gguf_model:+ for file in os.listdir(eval_folder):+ gguf_file = file+ user_model = AutoModelForCausalLM.from_pretrained(save_folder, gguf_file=gguf_file, device_map="auto")if hasattr(model, "hf_device_map") and len(model.hf_device_map) > 1:from accelerate.big_modeling import dispatch_model@@ -659,7 +671,7 @@ def eval(args):print(make_table(res))-def eval_task_by_task(model, device, tasks, batch_size=None, max_batch_size=64, trust_remote_code=True):+def eval_task_by_task(model, device, tasks, tokenizer=None, batch_size=None, max_batch_size=64, trust_remote_code=True):set_cuda_visible_devices(device)device_str, parallelism = get_device_and_parallelism(device)@@ -668,14 +680,30 @@ def eval_task_by_task(model, device, tasks, batch_size=None, max_batch_size=64,from auto_round.utils import loggerfrom lm_eval import simple_evaluate as lm_simple_evaluatefrom lm_eval.models.huggingface import HFLM+ from transformers import AutoModelForCausalLM, AutoTokenizer# from auto_round import AutoRoundConfigif batch_size is None:batch_size = "auto"+ is_gguf_file = Falseif not isinstance(model, str):parallelism = False+ else:+ if os.path.isfile(model) and model.endswith(".gguf"):+ is_gguf_file = True+ gguf_file = model+ model = os.path.dirname(model)+ else:+ for file in os.listdir(model):+ if file.endswith(".gguf"):+ is_gguf_file = True+ gguf_file = file+ if is_gguf_file:+ tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)+ model = AutoModelForCausalLM.from_pretrained(model, gguf_file=gguf_file, device_map="auto")hflm = HFLM(pretrained=model,+ tokenizer=tokenizer,device=device_str,batch_size=batch_size,max_batch_size=max_batch_size,diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 555618908..4e1a5d8a7 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -166,6 +166,11 @@ class WrapperLinear(torch.nn.Module):weight = self.orig_layer.get_weight().to(self.device)if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):weight = weight.t()++ quant_kwargs = {}+ if hasattr(self.orig_layer, "super_bits"):+ quant_kwargs["super_bits"] = self.orig_layer.super_bits+ quant_kwargs["super_group_size"] = self.orig_layer.super_group_sizeweight_q, scale, zp = self.weight_quant_func(weight,@@ -179,8 +184,8 @@ class WrapperLinear(torch.nn.Module):tensor_max=self.weight_max,data_type=self.data_type,q_scale_thresh=self.q_scale_thresh,-- )+ **quant_kwargs+ )weight_q = weight_q.to(weight.dtype)if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):weight_q = weight_q.t()@@ -237,6 +242,7 @@ class WrapperLinear(torch.nn.Module):self.orig_layer.to(self.device)##unwrapper weightqdq_weight, scale, zp = self._qdq_weight(v, min_scale, max_scale)+self.orig_layer.weight.data.copy_(qdq_weight)self.orig_layer.weight.grad = None@@ -244,19 +250,27 @@ class WrapperLinear(torch.nn.Module):if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):shape = qdq_weight.t().shape- if zp is not None:- zp = zp.reshape(shape[0], -1)- if isinstance(scale, dict):- for key in scale.keys():- if key == "scale":- name = key- self.orig_layer.scale = scale[key].reshape(shape[0], -1).to("cpu")+ def _set_dict_attr(attr_dict, attr_name):+ for key in attr_dict.keys():+ if key == attr_name:+ setattr(self.orig_layer, attr_name, attr_dict[key].reshape(shape[0], -1).to("cpu"))else:name = "w_" + key- setattr(self.orig_layer, name, scale[key].to("cpu"))+ setattr(self.orig_layer, name, attr_dict[key].to("cpu"))++ if isinstance(scale, dict):+ _set_dict_attr(scale, "scale")else:self.orig_layer.scale = scale.reshape(shape[0], -1).to("cpu")- self.orig_layer.zp = zp.to("cpu") if zp is not None else None++ if zp is not None:+ if isinstance(zp, dict):+ _set_dict_attr(zp, "zp")+ else:+ zp = zp.reshape(shape[0], -1)+ self.orig_layer.zp = zp.to("cpu") if zp is not None else None+ else:+ self.orig_layer.zp = None##unwrapper biasif self.enable_norm_bias_tuning and "bias_v" in best_params.keys(): ##fake quant