PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
Support for more gguf format and float zp for Q*_1 (#560)
Heng Guob38d19dc1039
diff --git a/.azure-pipelines/scripts/ut/run_ut.sh b/.azure-pipelines/scripts/ut/run_ut.shindex 91509b257..59dace8f9 100644--- a/.azure-pipelines/scripts/ut/run_ut.sh+++ b/.azure-pipelines/scripts/ut/run_ut.sh@@ -6,7 +6,7 @@ echo "set up UT env..."export TQDM_MININTERVAL=60export TQDM_POSITION=-1pip install pytest-cov pytest-html-pip install -r /auto-round/test/requirements.txt+pip install -r /auto-round/test/requirements.txt --extra-index-url https://download.pytorch.org/whl/cpupip list# install latest gguf for ut testgit clone https://github.com/ggml-org/llama.cpp.git && cd llama.cpp/gguf-py && pip install .diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 12577a29a..b527f16bf 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -52,6 +52,7 @@ from .utils import (llm_load_model,reset_params,init_cache, check_skippable_keywords, get_shared_keys, supported_dtypes, infer_bits_by_data_type,+ _gguf_args_check)from .low_cpu_mem.utils import get_layers_before_block@@ -225,7 +226,7 @@ class AutoRound(object):self.act_dynamic = act_dynamicself.act_data_type = act_data_typeif self.act_data_type is None:- if data_type in supported_dtypes and self.act_bits <= 16:+ if data_type in supported_dtypes and self.act_bits < 16:self.act_data_type = data_typelogger.info(f"activation adopts {data_type}")else:@@ -294,7 +295,6 @@ class AutoRound(object):"data_type","enable_quanted_input","enable_minmax_tuning",- "data_type","seqlen","batch_size","scale_dtype",@@ -452,7 +452,8 @@ class AutoRound(object):ValueError: If an unsupported format is specified."""# Validate and process the specified formats- formats = format.replace(' ', '').split(',')+ _gguf_args_check(self, format)+ formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')from auto_round.utils import supported_formatsfor format_ in formats:if format_ not in supported_formats:@@ -1193,8 +1194,8 @@ class AutoRound(object):if q_inputs is not None:q_inputs[i] = q_inputs[i].to(layer.weight.dtype)- wrapper_linear = WrapperLinear(layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(- device)+ wrapper_linear = WrapperLinear(+ layer, enable_minmax_tuning=self.enable_minmax_tuning, device=device).to(device)round_params = []minmax_params = []for key in wrapper_linear.params.keys():@@ -1623,6 +1624,7 @@ class AutoRound(object):Returns:object: The compressed model object."""+ format = format.replace("q*_", f"q{self.bits}_")# only support to export afp8if self.act_bits <= 8:if "fp8" not in self.act_data_type or self.act_dynamic:diff --git a/auto_round/data_type/int.py b/auto_round/data_type/int.pyindex 396053648..a16a730f1 100644--- a/auto_round/data_type/int.py+++ b/auto_round/data_type/int.py@@ -72,6 +72,73 @@ def double_quant_tensor(tensor, bits, q_scale_thresh):qdq_tensor = torch.clamp(round_ste(tensor / scale), max=maxq) * scalereturn qdq_tensor, scale+def double_quant_tensor_sym(tensor, bits, q_scale_thresh):+ maxq = 2 ** (bits - 1)+ imax = abs(tensor).argmax(axis=-1, keepdims=True)+ wmax = torch.take_along_dim(tensor, imax, axis=-1)+ scale = wmax / -maxq+ qdq_tensor = torch.where(scale != 0, round_ste(tensor / scale), 0).clip(-maxq, maxq -1) * scale+ return qdq_tensor, scale++@register_dtype("int_sym_dq")+def quant_tensor_sym_dq(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ super_group_size=16,+ super_bits=6,+ **kwargs):+ """Quantize and de-quantize tensor asymmetrically. full range, credict goes to llamacpp community++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)++ maxq = 2 ** (bits - 1)+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max++ wmin_abs = -(wmin_tmp * min_scale) # pylint: disable=E1130+ wmax_abs = wmax_tmp * max_scale+ max_v = (2 * (wmax_abs < wmin_abs).int() - 1) * torch.max(wmax_abs, wmin_abs)+ scale = (max_v / maxq).to(scale_dtype)+ scale = torch.where(scale < 0, torch.clamp(scale, max=-q_scale_thresh), torch.clamp(scale, min=q_scale_thresh))+ scale = scale.view(-1, super_group_size)++ #conduct double quant+ scale, d_scale = double_quant_tensor_sym(scale, super_bits, q_scale_thresh)++ scale = scale.view(-1, 1)+ zp = torch.full_like(scale, maxq) # pylint: disable=E1130+ int_w = round_ste(tensor / scale + v)+ q = torch.clamp(int_w + zp, 0, 2 ** bits - 1)+ qdq_result = (scale * (q - zp)).to(tensor.dtype)+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, {"scale": scale, "d_scale": d_scale}, zp@register_dtype("int_asym_dq")def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,@@ -177,6 +244,62 @@ def quant_tensor_asym(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scareturn qdq_result, scale, zp+@register_dtype("int_asym_float_zp")+def quant_tensor_asym_float_zp(+ tensor,+ bits=4,+ group_size=-1,+ v=0,+ min_scale=1.0,+ max_scale=1.0,+ scale_dtype=torch.float16,+ tensor_min=None,+ tensor_max=None,+ q_scale_thresh=1e-5,+ **kwargs):+ """Quantize and de-quantize tensor asymmetrically.++ Args:+ tensor: Tensor containing the tensor to be quantized+ bits: Number of bits for quantization (e.g., 2, 3, 4, 8)+ group_size: Number of elements to share scale for quantization+ v: Rounding value perturbation+ min_scale: Minimum scale coefficient for tensor+ max_scale: Maximum scale coefficient for tensor+ tensor_min (Tensor, optional): Minimum tensor value for quantization. Defaults to None.+ tensor_max (Tensor, optional): Maximum tensor value for quantization. Defaults to None.+ scale_dtype: dtype of the quantized scale,as most kernels only support FP16 or FP32, while this value is import+ q_scale_thresh: clip the quantized scale's magnitude to this value to improve the numerical stability++ Returns:+ Quantized and de-quantized tensor, scale, zero-point+ """+ tensor, orig_shape, pad_len = reshape_pad_tensor_by_group_size(tensor, group_size)+ maxq = 2 ** bits - 1+ if tensor_min is None or tensor_max is None:+ wmin_tmp = torch.clamp(tensor.min(-1)[0], max=0)+ wmax_tmp = torch.clamp(tensor.max(-1)[0], min=0)+ else:+ wmin_tmp = tensor_min+ wmax_tmp = tensor_max+ if isinstance(min_scale, torch.Tensor):+ wmin = wmin_tmp * min_scale+ wmax = wmax_tmp * max_scale+ else:+ wmin = wmin_tmp+ wmax = wmax_tmp+ scale = ((wmax - wmin) / maxq).to(scale_dtype)+ scale = torch.clamp(scale, min=q_scale_thresh)+ zp = -wmin / scale # pylint: disable=E1130+ scale = scale.unsqueeze(dim=-1)+ zp = zp.unsqueeze(dim=-1)+ int_w = round_ste(tensor / scale + v)+ q = torch.clamp(int_w + zp, 0, maxq)+ qdq_result = (scale * (q - zp)).to(tensor.dtype)+ qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)+ return qdq_result, scale, zp++@register_dtype("int_sym_gptq")def quant_tensor_sym_gptq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale=1.0, scale_dtype=torch.float16,tensor_min=None,diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex 2d7b1f163..8cc3f85e2 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -14,20 +14,31 @@GGUF_CONFIG = {}-GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": False, "data_type": "int"}+GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}-GGUF_CONFIG["gguf:q4_1"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": True, "data_type": "int"}--GGUF_CONFIG["gguf:q4_k_s"] = {+GGUF_CONFIG["gguf:q4_1"] = {"bits": 4,"act_bits": 16,- "super_group_size": 8,- "super_bits": 6,"group_size": 32,"asym": True,- "data_type": "int_asym_dq"+ "sym": False,+ "data_type": "int_asym_float_zp"+}++GGUF_CONFIG["gguf:q5_0"] = {"bits": 5, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}++GGUF_CONFIG["gguf:q5_1"] = {+ "bits": 5,+ "act_bits": 16,+ "group_size": 32,+ "asym": True,+ "sym": False,+ "data_type": "int_asym_float_zp"}+GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}++GGUF_CONFIG["gguf:q2_k_s"] = {"bits": 2,"act_bits": 16,@@ -35,7 +46,50 @@ GGUF_CONFIG["gguf:q2_k_s"] = {"super_bits": 4,"group_size": 16,"asym": True,+ "sym": False,"data_type": "int_asym_dq"}-GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "asym": False, "data_type": "int"}+GGUF_CONFIG["gguf:q3_k_s"] = {+ "bits": 3,+ "act_bits": 16,+ "super_group_size": 16,+ "super_bits": 6,+ "group_size": 16,+ "asym": False,+ "sym": True,+ "data_type": "int_sym_dq"+}++GGUF_CONFIG["gguf:q4_k_s"] = {+ "bits": 4,+ "act_bits": 16,+ "super_group_size": 8,+ "super_bits": 6,+ "group_size": 32,+ "asym": True,+ "sym": False,+ "data_type": "int_asym_dq"+}++GGUF_CONFIG["gguf:q5_k_s"] = {+ "bits": 5,+ "act_bits": 16,+ "super_group_size": 8,+ "super_bits": 6,+ "group_size": 32,+ "asym": True,+ "sym": False,+ "data_type": "int_asym_dq"+}++GGUF_CONFIG["gguf:q6_k"] = GGUF_CONFIG["gguf:q6_k_s"] = {+ "bits": 6,+ "act_bits": 16,+ "super_group_size": 16,+ "super_bits": 8,+ "group_size": 16,+ "asym": False,+ "sym": True,+ "data_type": "int_sym_dq"+}diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 22c282ae1..6cdd5efd3 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -1155,11 +1155,12 @@ class Model(OriModel):scale = module.scalezp = module.zp if hasattr(module, "zp") else Noneggml_quant = ggml_quant_gpu if torch.cuda.is_available() else ggml_quant_cpu+ # ggml_quant = ggml_quant_cpuif data_qtype.name.lower().endswith("_k"):d_scale = module.w_d_scale.to(torch.float32)- d_wmin_m = module.w_d_wmin_m.to(torch.float32)- wmin_m = module.w_wmin_m.to(torch.float32)+ d_wmin_m = module.w_d_wmin_m.to(torch.float32) if hasattr(module, "w_d_wmin_m") else None+ wmin_m = module.w_wmin_m.to(torch.float32) if hasattr(module, "w_wmin_m") else Nonedata = ggml_quant(data_torch,data_qtype.name.lower(),@@ -1255,10 +1256,20 @@ class Model(OriModel):data_qtype = gguf.GGMLQuantizationType.Q4_0elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_1:data_qtype = gguf.GGMLQuantizationType.Q4_1- elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:- data_qtype = gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_0:+ data_qtype = gguf.GGMLQuantizationType.Q5_0+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_1:+ data_qtype = gguf.GGMLQuantizationType.Q5_1elif self.ftype == gguf.LlamaFileType.MOSTLY_Q2_K_S:data_qtype = gguf.GGMLQuantizationType.Q2_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q3_K_S:+ data_qtype = gguf.GGMLQuantizationType.Q3_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:+ data_qtype = gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:+ data_qtype = gguf.GGMLQuantizationType.Q5_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:+ data_qtype = gguf.GGMLQuantizationType.Q6_Kelse:raise ValueError(f"Unknown file type: {self.ftype.name}")diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex 81a56616b..5d6d3459c 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -23,7 +23,6 @@ import timegguf = LazyImport("gguf")-FTYPE_MAP: dict[str, gguf.LlamaFileType] = {"f32": gguf.LlamaFileType.ALL_F32,"f16": gguf.LlamaFileType.MOSTLY_F16,@@ -31,9 +30,15 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {"q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,"q4_0": gguf.LlamaFileType.MOSTLY_Q4_0,"q4_1": gguf.LlamaFileType.MOSTLY_Q4_1,- "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,- "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,+ "q5_0": gguf.LlamaFileType.MOSTLY_Q5_0,+ "q5_1": gguf.LlamaFileType.MOSTLY_Q5_1,"q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,+ "q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,+ "q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,+ "q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,+ "q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,+ "q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,+ "q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,"auto": gguf.LlamaFileType.GUESSED,}diff --git a/auto_round/export/export_to_gguf/quant_cpu.py b/auto_round/export/export_to_gguf/quant_cpu.pyindex 24b20e94b..563284017 100644--- a/auto_round/export/export_to_gguf/quant_cpu.py+++ b/auto_round/export/export_to_gguf/quant_cpu.py@@ -13,27 +13,15 @@# limitations under the License.import numpy as np-from tqdm import tqdmfrom concurrent.futures import ProcessPoolExecutor+from .utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES-QK_K = 256-K_SCALE_SIZE = 12-GGML_QUANT_SIZES = {- "bf16": (1, 2),- "q4_0": (32, 2 + 16),- "q4_1": (32, 2 + 2 + 16),- "q4_k": (256, 2 + 2 + QK_K//2 + 12),- "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),- "q8_0": (32, 2 + 32)-}+GGML_QUANT_TYPE = {}-GGML_QUANT_BLOCK = {}---def register_block(name):+def register_qtype(name):def register(cls):- GGML_QUANT_BLOCK[name] = cls+ GGML_QUANT_TYPE[name] = clsreturn clsreturn register@@ -55,37 +43,8 @@ def ggml_quant_cpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=Non_blocks = data.size // block_sizeblocks = data.reshape((n_blocks, block_size))- if ggml_type.endswith("_k"):- worker = 16- else:- worker = 0-- worker = 0- if worker > 0:- n_groups = (data.shape[0] // worker) or 1- blocks = np.array_split(blocks, n_groups, axis=0)- scale = np.array_split(scale, n_groups, axis=0) if scale is not None else [None] * n_groups- zp = np.array_split(zp, n_groups, axis=0) if zp is not None else [None] * n_groups- wmin_m = np.array_split(wmin_m, n_groups, axis=0) if wmin_m is not None else [None] * n_groups- d_scale = np.array_split(d_scale, n_groups, axis=0) if d_scale is not None else [None] * n_groups- d_wmin_m = np.array_split(d_wmin_m, n_groups, axis=0) if d_wmin_m is not None else [None] * n_groups-- quant_func = GGML_QUANT_BLOCK[ggml_type]- if ggml_type.endswith("_k"):- with ProcessPoolExecutor(worker) as executor:- result = executor.map(quant_func, blocks, scale, zp, wmin_m, d_scale, d_wmin_m)- new_data = np.array(list(result), dtype=np.uint8)- else:- with ProcessPoolExecutor(n_groups) as executor:- result = executor.map(quant_func, blocks, scale, zp)- new_data = np.array(list(result), dtype=np.uint8)- else:- quant_func = GGML_QUANT_BLOCK[ggml_type]- if ggml_type.endswith("_k"):- # 0.5588 vs 63.05- new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)- else:- new_data = quant_func(blocks, scale, zp)+ quant_func = GGML_QUANT_TYPE[ggml_type]+ new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)assert new_data.dtype == np.uint8assert new_data.shape[-1] == type_size@@ -100,8 +59,170 @@ def np_roundf(n: np.ndarray) -> np.ndarray:return np.sign(n) * b-@register_block("bf16")-def bf16_quant_block(blocks: np.array, scale=None, zp=None):+def make_qx_quant(data, bits, rmse_type=0, qw = None):+ nmax = pow(2, bits - 1)+ imax = abs(data).argmax(axis=-1, keepdims=True)+ group_max = np.take_along_axis(data, imax, axis=-1)+ iscales = np.where(group_max != 0, -nmax / group_max, 0)++ if rmse_type == 0:+ L = (np.round(iscales * data).clip(-nmax, nmax-1) + nmax).astype(np.uint8)+ scales = np.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])+ return scales, L++ return_early = False+ if rmse_type < 0:+ rmse_type = -rmse_type+ return_early = True++ L = np.round(iscales * data).clip(-nmax, nmax-1)+ if qw is not None:+ w = qw+ elif rmse_type == 1:+ w = data ** 2+ elif rmse_type == 2:+ w = 1+ elif rmse_type == 3:+ w = np.fabs(data)+ else:+ w = np.sqrt(np.fabs(data))+ sumlx = np.sum(w * data * L, axis=-1)+ suml2 = np.sum(w * L * L, axis=-1)+ scales = np.where(suml2 !=0, sumlx / suml2, 0)+ if return_early:+ iscales_inv = np.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])+ scales = np.where(suml2 > 0, 0.5 * (scales + iscales_inv), iscales_inv)+ L = (L + nmax).astype(np.uint8)+ return scales, L+ best = scales * sumlx+ for _is in range(-9, 10):+ if _is == 0:+ continue+ iscales = np.where(group_max != 0, -(nmax + -0.1 * _is) / nmax, 0)+ tmp_L = np.round(iscales * data).clip(-nmax, nmax - 1)+ sumlx = np.sum(w * data * L, axis=-1)+ suml2 = np.sum(w * L * L, axis=-1)++ replace_id = (suml2 > 0) & (sumlx*sumlx > best*suml2)+ L[replace_id] = tmp_L[replace_id]+ scales[replace_id] = sumlx[replace_id] / suml2[replace_id]+ best[replace_id] = scales[replace_id] * sumlx[replace_id]++ L = (L + nmax).astype(np.uint8)+ return scales, L+++def make_q3_quants(data, bits, do_rmse=False):+ # data shape (nb, 16, 16)+ nmax = pow(2, bits - 1)+ imax = abs(data).argmax(axis=-1, keepdims=True)+ group_max = np.take_along_axis(data, imax, axis=-1)+ iscale = np.where(group_max != 0, -nmax / group_max, 0)+ if do_rmse:+ L = np.round(iscale * data).clip(-nmax, nmax - 1)+ w = np.power(data, 2)+ sumlx = np.sum(w * data * L, axis=-1)+ suml2 = np.sum(w * L * L, axis=-1)++ for itry in range(5):+ # w = np.power(data, 2)+ if len(sumlx.shape) == 2:+ sumlx = sumlx.reshape(*sumlx.shape, 1)+ if len(suml2.shape) == 2:+ suml2 = suml2.reshape(*suml2.shape, 1)+ slx = sumlx - w*data*L+ repalce_idx = slx > 0+ sl2 = suml2 - w*L*L+ tmp_L = np.where(slx > 0, data * sl2 / slx, 0)+ new_L = np.round(tmp_L).clip(-nmax, nmax - 1)+ tmp_repalce_idx = repalce_idx & (new_L != L)+ slx[tmp_repalce_idx] += w[tmp_repalce_idx] * data[tmp_repalce_idx] * new_L[tmp_repalce_idx]+ sl2[tmp_repalce_idx] += w[tmp_repalce_idx] * new_L[tmp_repalce_idx] * new_L[tmp_repalce_idx]+ repalce_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)++ L[repalce_idx] = new_L[repalce_idx]+ sumlx = slx+ suml2 = sl2++ L = np.round(iscale * data).clip(-nmax, nmax - 1) + nmax+ L = L.astype(np.uint8)+ scales = np.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])+ return scales, L+++def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):+ # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K+ nmax = pow(2, bits) - 1+ if len(data.shape) == 2:+ if bits in [4, 5]:+ data_shape = (-1, 8, 32)+ elif bits in [2, 3]:+ data_shape = (-1, 16, 16)+ else:+ raise NotImplementedError(f"bits = {bits} is not supported")+ data = data.reshape(data_shape)+ sum_x2 = np.sum(np.power(data, 2), axis=-1, keepdims=True)+ av_x = np.sqrt(sum_x2 / 32)+ weight = np.abs(data) + av_x++ group_min = np.min(data, axis=-1, keepdims=True)+ group_max = np.max(data, axis=-1, keepdims=True)++ the_mins = -group_min++ sum_w = np.sum(weight, axis=-1, keepdims=True)+ sum_x = np.sum(weight * data, axis=-1, keepdims=True)++ group_min[group_min > 0] = 0++ scale = (group_max-group_min) / nmax+ iscale = np.where(scale == 0, 0, 1 / scale)++ l_values = np.round(iscale * (data-group_min))+ L = np.clip(l_values, 0, nmax).astype(np.uint8)++ diffs = scale*L + group_min - data+ diffs = np.abs(diffs) if use_mad else diffs**2+ best_mad = np.sum(weight * diffs, axis=-1, keepdims=True)++ if nstep < 1:+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])++ for step in range(nstep):+ new_scale = (group_max-group_min) / (rmin + rdelta*step + nmax)+ new_iscale = np.where(new_scale == 0, 0, 1 / new_scale)++ l_values = np.round(new_iscale * (data-group_min))+ Laux = np.clip(l_values, 0, nmax).astype(np.uint8)++ sum_l = np.sum(weight * Laux, axis=-1, keepdims=True)+ sum_l2 = np.sum(weight * Laux**2, axis=-1, keepdims=True)+ sum_xl = np.sum(weight * Laux * data, axis=-1, keepdims=True)++ D = sum_w*sum_l2 - sum_l*sum_l+ replace_idx = D > 0++ this_scale = (sum_w*sum_xl - sum_x*sum_l) / D+ this_min = (sum_l2*sum_x - sum_l*sum_xl) / D+ this_min[this_min > 0] = 0+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]++ diffs = this_scale*Laux + this_min - data+ diffs = np.abs(diffs) if use_mad else diffs**2+ mad = np.sum(weight * diffs, axis=-1, keepdims=True)++ replace_idx &= mad < best_mad+ best_mad[replace_idx] = mad[replace_idx]+ L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]+ scale[replace_idx] = this_scale[replace_idx]+ group_min[replace_idx] = this_min[replace_idx]++ the_mins = -group_min+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])+++@register_qtype("bf16")+def bf16_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):n = blocks.view(np.uint32)# force nan to quietn = np.where((n & 0x7fffffff) > 0x7f800000, (n & np.uint32(0xffff0000)) | np.uint32(64 << 16), n)@@ -110,8 +231,8 @@ def bf16_quant_block(blocks: np.array, scale=None, zp=None):return n.astype(np.uint16).view(np.uint8)-@register_block("q4_0")-def q4_0_quant_block(blocks: np.array, scale=None, zp=None):+@register_qtype("q4_0")+def q4_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):if scale is not None:d = scale.reshape((-1, 1))else:@@ -134,8 +255,8 @@ def q4_0_quant_block(blocks: np.array, scale=None, zp=None):return np.concatenate([d, qs], axis=-1)-@register_block("q4_1")-def q4_1_quant_block(blocks: np.array, scale=None, zp=None):+@register_qtype("q4_1")+def q4_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):if scale is not None:d = scale.reshape((-1, 1))min = zp.reshape((-1, 1)) * d * -1@@ -158,106 +279,93 @@ def q4_1_quant_block(blocks: np.array, scale=None, zp=None):return np.concatenate([d, m, qs], axis=-1)-@register_block("q8_0")-def q8_0_quant_block(blocks: np.array, scale=None, zp=None) -> np.ndarray:+@register_qtype("q5_0")+def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):if scale is not None:d = scale.reshape((-1, 1))else:- d = abs(blocks).max(axis=1, keepdims=True) / 127+ imax = abs(blocks).argmax(axis=-1, keepdims=True)+ max = np.take_along_axis(blocks, imax, axis=-1)+ d = max / -16++ n_blocks = blocks.shape[0]+ block_size = GGML_QUANT_SIZES["q5_0"][0]with np.errstate(divide="ignore"):id = np.where(d == 0, 0, 1 / d)- qs = np_roundf(blocks * id)-- # (n_blocks, 2)- d = d.astype(np.float16).view(np.uint8)- # (n_blocks, block_size)- qs = qs.astype(np.int8).view(np.uint8)-- return np.concatenate([d, qs], axis=1)--def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):- assert bits in [2, 4], f"bits={bits} is not supported"- nmax = bits ** 2 -1- # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K- if len(data.shape) == 2:- data_shape = (-1, 8, 32) if bits == 4 else (-1, 16, 16)- data = data.reshape(data_shape)- sum_x2 = np.sum(np.power(data, 2), axis=-1, keepdims=True)- av_x = np.sqrt(sum_x2 / 32)- weight = np.abs(data) + av_x+ # FIXME: Q5_0's reference rounding is cursed and depends on FMA+ q = np.trunc((np.float64(blocks) * np.float64(id)) + np.float64(16.5),+ dtype=np.float32).astype(np.uint8).clip(0, 31)- group_min = np.min(data, axis=-1, keepdims=True)- group_max = np.max(data, axis=-1, keepdims=True)+ qs = q.reshape((n_blocks, 2, block_size // 2))+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))- the_mins = -group_min+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)- sum_w = np.sum(weight, axis=-1, keepdims=True)- sum_x = np.sum(weight * data, axis=-1, keepdims=True)+ d = d.astype(np.float16).view(np.uint8)- group_min[group_min > 0] = 0+ return np.concatenate([d, qh, qs], axis=-1)- scale = (group_max - group_min) / nmax- iscale = np.where(scale == 0, 0, 1 /scale)- l_values = np.round(iscale * (data - group_min))- L = np.clip(l_values, 0, nmax).astype(np.uint8)+@register_qtype("q5_1")+def q5_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):+ if scale is not None:+ d = scale.reshape((-1, 1))+ min = zp.reshape((-1, 1)) * d * -1+ else:+ max = blocks.max(axis=-1, keepdims=True)+ min = blocks.min(axis=-1, keepdims=True)+ d = (max-min) / 31- diffs = scale * L + group_min - data- diffs = np.abs(diffs) if use_mad else diffs**2- best_mad = np.sum(weight * diffs, axis=-1, keepdims=True)+ n_blocks = blocks.shape[0]+ block_size = GGML_QUANT_SIZES["q5_1"][0]- if nstep < 1:- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ q = np.trunc((blocks-min) * id + np.float32(0.5), dtype=np.float32).astype(np.uint8).clip(0, 31)- for step in range(nstep):- new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)- new_iscale = np.where(new_scale == 0, 0, 1 /new_scale)+ qs = q.reshape((n_blocks, 2, block_size // 2))+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))- l_values = np.round(new_iscale * (data - group_min))- Laux = np.clip(l_values, 0, nmax).astype(np.uint8)+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)- sum_l = np.sum(weight * Laux, axis=-1, keepdims=True)- sum_l2 = np.sum(weight * Laux**2, axis=-1, keepdims=True)- sum_xl = np.sum(weight * Laux * data, axis=-1, keepdims=True)+ d = d.astype(np.float16).view(np.uint8)+ m = min.astype(np.float16).view(np.uint8)- D = sum_w * sum_l2 - sum_l * sum_l- replace_idx = D > 0+ return np.concatenate([d, m, qh, qs], axis=-1)- this_scale = (sum_w * sum_xl - sum_x * sum_l) / D- this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D- this_min[this_min > 0] = 0- this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]- diffs = this_scale * Laux + this_min - data- diffs = np.abs(diffs) if use_mad else diffs**2- mad = np.sum(weight * diffs, axis=-1, keepdims=True)+@register_qtype("q8_0")+def q8_0_quant_block(blocks: np.array, scale=None, zp=None) -> np.ndarray:+ if scale is not None:+ d = scale.reshape((-1, 1))+ else:+ d = abs(blocks).max(axis=1, keepdims=True) / 127+ with np.errstate(divide="ignore"):+ id = np.where(d == 0, 0, 1 / d)+ qs = np_roundf(blocks * id)- replace_idx &= mad < best_mad- best_mad[replace_idx] = mad[replace_idx]- L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]- scale[replace_idx] = this_scale[replace_idx]- group_min[replace_idx] = this_min[replace_idx]+ # (n_blocks, 2)+ d = d.astype(np.float16).view(np.uint8)+ # (n_blocks, block_size)+ qs = qs.astype(np.int8).view(np.uint8)- the_mins = -group_min- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])+ return np.concatenate([d, qs], axis=1)-@register_block("q2_k")+@register_qtype("q2_k")def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):nb = blocks.shape[0]output_scale = np.empty((nb, 16), dtype=np.uint8)output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)- blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)+ blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)if scale is not None:scales = scale.reshape((-1, QK_K // 16))mins = wmin_m.reshape((-1, QK_K // 16))- output_d = d_scale.astype(np.float32) if len(d_scale.shape) == 2 \- else d_scale.reshape(-1, 1).astype(np.float32)- output_dmin = d_wmin_m.astype(np.float32) if len(d_wmin_m.shape) == 2 \- else d_wmin_m.reshape(-1, 1).astype(np.float32)- inv_scales = np.where(d_scale == 0, 0, 1 / output_d)+ output_d = d_scale.reshape(-1, 1).astype(np.float32)+ output_dmin = d_wmin_m.reshape(-1, 1).astype(np.float32)+ inv_scales = np.where(output_d == 0, 0, 1 / output_d)inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)@@ -268,7 +376,7 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scalemax_mins = np.max(mins, axis=-1, keepdims=True)inv_scales = np.where(max_scales > 0, 15. / max_scales, 0)inv_mins = np.where(max_mins > 0, 15. / max_mins, 0)-+replace_ids = (max_scales > 0).reshape(-1)output_scale = np.zeros_like(scales).astype(np.uint8)output_scale[replace_ids] = np.round(inv_scales * scales).astype(np.uint8)@@ -279,8 +387,8 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleif d_scale is None:output_d = np.where(max_scales > 0, max_scales / 15, 0)if d_wmin_m is None:- output_dmin = np.where(max_mins > 0 ,max_mins / 15., 0)-+ output_dmin = np.where(max_mins > 0, max_mins / 15., 0)+d_tmp = output_d * (output_scale & 0xF)dm_tmp = output_dmin * (output_scale >> 4)@@ -290,19 +398,73 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scalenp.uint8)all_L = np.clip(all_L, 0, 3)- output_qs = all_L[:,::4] | (all_L[:,1::4] << 2) | (all_L[:,2::4] << 4) | (all_L[:,3::4] << 6)+ output_qs = all_L[:, ::4] | (all_L[:, 1::4] << 2) | (all_L[:, 2::4] << 4) | (all_L[:, 3::4] << 6)output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)output_qs = output_qs.reshape((nb, QK_K // 4))# [scale, qs, d, dmin]return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)--@register_block("q4_k")++@register_qtype("q3_k")+def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ blocks = blocks.reshape((nb, QK_K // 16, 16))++ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)++ if scale is not None:+ scales = scale.reshape(-1, QK_K // 16)+ output_d = d_scale.reshape(-1, 1).astype(np.float32)+ iscales = np.where(output_d == 0, 0, 1 / output_d)+ all_L = np.round(blocks / scales.reshape(*scales.shape, 1)).clip(-4, 3) + 4+ all_L = all_L.astype(np.uint8)+ else:+ scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=False)+ imax = abs(scales).argmax(axis=-1, keepdims=True)+ max_scales = np.take_along_axis(scales, imax, axis=-1)+ iscales = np.where(max_scales != 0, -32 / max_scales, 0)+ output_d = np.where(iscales != 0, 1 / iscales, 0)++ q_scales = np.round(iscales * scales).clip(-32, 31) + 32+ q_scales = q_scales.astype(np.uint8)+ output_scale[:, :8] = (q_scales[:, :8] & 0xF) | ((q_scales[:, 8:] & 0xF) << 4)+ hmask = q_scales >> 4+ output_scale[:, 8:] = hmask[:, :4] | hmask[:, 4:8] << 2 | hmask[:, 8:12] << 4 | hmask[:, 12:] << 6+++ # lscales, hscales = np.hsplit(output_scale, [8])+ # lscales = lscales.reshape((nb, 1, 8)) >> np.array([0, 4], dtype=np.uint8).reshape((1, 2, 1))+ # lscales = lscales.reshape((nb, 16))+ # hscales = hscales.reshape((nb, 1, 4)) >> np.array([0, 2, 4, 6], dtype=np.uint8).reshape((1, 4, 1))+ # hscales = hscales.reshape((nb, 16))+ # sc = (lscales & np.uint8(0x0F)) | ((hscales & np.uint8(0x03)) << np.uint8(4))+ # sc = (sc.astype(np.int8) - np.int8(32)).astype(np.float32)+ sc = np.round(iscales * scales).clip(-32, 31)++ d = output_d.astype(np.float32) * sc+ replace_idx = (d != 0)++ all_L[replace_idx] = np.round(blocks[replace_idx] / d[replace_idx].reshape(-1, 1)).clip(-4, 3) + 4++ output_hmask = np.bitwise_or.reduce(+ all_L.reshape(nb, 8, 32) >> 2 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1) # pylint: disable=E1121+ all_L = np.where(all_L > 3, all_L - 4, all_L)++ output_qs = np.bitwise_or.reduce(+ all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121++ output_qs = output_qs.reshape(nb, 64).astype(np.uint8)+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)+ # [hmask, qs, scale, d]+ return np.concatenate([output_hmask, output_qs, output_scale, output_d], axis=-1)+++@register_qtype("q4_k")def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):nb = blocks.shape[0]- blocks = blocks.reshape((nb, QK_K // 32, 32)) # (nb, 8, 32)+ blocks = blocks.reshape(nb, QK_K // 32, 32) # (nb, 8, 32)output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)@@ -310,11 +472,9 @@ def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleif scale is not None:scales = scale.reshape(-1, QK_K // 32)mins = wmin_m.reshape(-1, QK_K // 32)- output_d = d_scale.astype(np.float32) if len(d_scale.shape) == 2 \- else d_scale.reshape(-1, 1).astype(np.float32)- output_dmin = d_wmin_m.astype(np.float32) if len(d_wmin_m.shape) == 2 \- else d_wmin_m.reshape(-1, 1).astype(np.float32)- inv_scales = np.where(d_scale == 0, 0, 1 / output_d)+ output_d = d_scale.reshape(-1, 1).astype(np.float32)+ output_dmin = d_wmin_m.reshape(-1, 1).astype(np.float32)+ inv_scales = np.where(output_d == 0, 0, 1 / output_d)inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)@@ -326,15 +486,15 @@ def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleinv_scales = np.where(max_scales > 0, 63. / max_scales, 0)inv_mins = np.where(max_mins > 0, 63. / max_mins, 0)- q_scales = np.round(inv_scales * scales).astype(np.uint8)- q_mins = np.round(inv_mins * mins).astype(np.uint8)+ q_scales = np.round(inv_scales * scales).astype(np.uint8).clip(0, 63)+ q_mins = np.round(inv_mins * mins).astype(np.uint8).clip(0, 63)- output_scale[:,:4] = q_scales[:,:4]- output_scale[:,4:8] = q_mins[:,:4]+ output_scale[:, :4] = q_scales[:, :4]+ output_scale[:, 4:8] = q_mins[:, :4]- output_scale[:,8:] = (q_scales[:,4:] & 0xF) | ((q_mins[:,4:] & 0xF) << 4)- output_scale[:,:4] |= ((q_scales[:,4:] >> 4) << 6)- output_scale[:,4:8] |= ((q_mins[:,4:] >> 4) << 6)+ output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)+ output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)+ output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)if d_scale is None:output_d = max_scales / 63@@ -349,7 +509,7 @@ def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale(blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(np.uint8)all_L = np.clip(all_L, 0, 15)- output_qs = all_L[:,::2] | (all_L[:,1::2] << 4)+ output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)@@ -357,3 +517,99 @@ def q4_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale# [d, dmin, scale, qs]return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)+++@register_qtype("q5_k")+def q5_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ blocks = blocks.reshape((nb, QK_K // 32, 32))++ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)+ output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)+ output_qh = np.empty((nb, QK_K // 8), dtype=np.uint8)++ if scale is not None:+ scales = scale.reshape(-1, QK_K // 32)+ mins = wmin_m.reshape(-1, QK_K // 32)+ output_d = d_scale.reshape(-1, 1).astype(np.float32)+ output_dmin = d_wmin_m.reshape(-1, 1).astype(np.float32)+ inv_scales = np.where(output_d == 0, 0, 1 / output_d)+ inv_mins = np.where(d_wmin_m == 0, 0, 1 / output_dmin)+ max_scales = np.max(scales, axis=-1, keepdims=True) # (nb, 1)+ max_mins = np.max(mins, axis=-1, keepdims=True) # (nb, 1)+ all_L = np.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).astype(np.uint8)+ else:+ scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)+ max_scales = np.max(scales, axis=-1, keepdims=True)+ max_mins = np.max(mins, axis=-1, keepdims=True)+ inv_scales = np.where(max_scales > 0, 63. / max_scales, 0)+ inv_mins = np.where(max_mins > 0, 63. / max_mins, 0)++ q_scales = np.round(inv_scales * scales).astype(np.uint8).clip(0, 63)+ q_mins = np.round(inv_mins * mins).astype(np.uint8).clip(0, 63)++ output_scale[:, :4] = q_scales[:, :4]+ output_scale[:, 4:8] = q_mins[:, :4]++ output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)+ output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)+ output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)++ if d_scale is None:+ output_d = max_scales / 63+ if d_wmin_m is None:+ output_dmin = max_mins / 63++ d_tmp = output_d * q_scales+ dm_tmp = output_dmin * q_mins++ replace_ids = d_tmp != 0+ all_L[replace_ids] = np.round(+ (blocks[replace_ids] + dm_tmp[replace_ids].reshape(-1, 1)) / d_tmp[replace_ids].reshape(-1, 1)).astype(+ np.uint8)+ all_L = np.clip(all_L, 0, 31) # (nb, 8, 32)++ output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)+ output_qh = np.bitwise_or.reduce(+ all_L >> 4 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1).astype(np.uint8) # pylint: disable=E1121++ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_qs = output_qs.reshape(nb, QK_K // 2)+ # [d, dmin, scale, qh, qs]+ return np.concatenate([output_d, output_dmin, output_scale, output_qh, output_qs], axis=-1)++@register_qtype("q6_k")+def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ blocks = blocks.reshape((nb, QK_K // 16, 16))++ if scale is not None:+ scales = scale.reshape(-1, QK_K // 16)+ output_d = d_scale.reshape(-1, 1).astype(np.float32)+ iscales = np.where(output_d == 0, 0, 1 / output_d)+ all_L = (np.round(blocks / scales.reshape(*scales.shape, 1)).clip(-32, 31) + 32).astype(np.uint8)+ else:+ scales, all_L = make_qx_quant(blocks, bits=6, rmse_type=1, qw=None)+ imax = abs(scales).argmax(axis=-1, keepdims=True)+ max_scales = np.take_along_axis(scales, imax, axis=-1)+ iscales = np.where(max_scales != 0, -128 / max_scales, 0)+ output_d = np.where(iscales != 0, 1 / iscales, -1)++ output_scale = np.round(iscales * scales).clip(max=127).astype(np.int8)+ d_tmp = output_d * output_scale.astype(np.float32)+ replace_ids = d_tmp != 0+ all_L[replace_ids] = np.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \+ .clip(0, 63).astype(np.uint8)++ tmp_L = all_L.reshape(nb, 4, 64) & 0xF+ output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2)+ output_qh = np.bitwise_or.reduce(+ (all_L >> 4).reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), # pylint: disable=E1121+ axis=2).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121++ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)++ # [ql, qh, scale, d]+ output_scale = output_scale.view(np.uint8)+ return np.concatenate([output_ql, output_qh, output_scale, output_d], axis=-1)diff --git a/auto_round/export/export_to_gguf/quant_gpu.py b/auto_round/export/export_to_gguf/quant_gpu.pyindex 7d669bfda..57407e021 100644--- a/auto_round/export/export_to_gguf/quant_gpu.py+++ b/auto_round/export/export_to_gguf/quant_gpu.py@@ -15,26 +15,15 @@import torchimport numpy as np-QK_K = 256-K_SCALE_SIZE = 12-GGML_QUANT_SIZES = {- "bf16": (1, 2),- "q4_0": (32, 2 + 16),- "q4_1": (32, 2 + 2 + 16),- "q4_k": (256, 2 + 2 + QK_K//2 + 12),- "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),- "q8_0": (32, 2 + 32)-}+from .utils import QK_K, K_SCALE_SIZE, GGML_QUANT_SIZES-GGML_QUANT_BLOCK = {}+GGML_QUANT_TYPE = {}---def register_block(name):+def register_qtype(name):def register(cls):- GGML_QUANT_BLOCK[name] = cls+ GGML_QUANT_TYPE[name] = clsreturn clsreturn register@@ -54,7 +43,7 @@ def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=Non_blocks = data.nelement() // block_sizeblocks = data.reshape((n_blocks, block_size))- quant_func = GGML_QUANT_BLOCK[ggml_type]+ quant_func = GGML_QUANT_TYPE[ggml_type]new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m)assert new_data.shape[-1] == type_size@@ -69,7 +58,166 @@ def torch_roundf(n):return torch.sign(n) * b-@register_block("bf16")+def make_qx_quant(data, bits, rmse_type=0, qw = None):+ nmax = pow(2, bits - 1)+ imax = abs(data).argmax(axis=-1, keepdims=True)+ group_max = torch.take_along_dim(data, imax, axis=-1)+ iscales = torch.where(group_max != 0, -nmax / group_max, 0)++ if rmse_type == 0:+ L = (torch.round(iscales * data).clip(-nmax, nmax-1) + nmax).to(torch.uint8)+ scales = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])+ return scales, L++ return_early = False+ if rmse_type < 0:+ rmse_type = -rmse_type+ return_early = True++ L = torch.round(iscales * data).clip(-nmax, nmax-1)+ if qw is not None:+ w = qw+ elif rmse_type == 1:+ w = data ** 2+ elif rmse_type == 2:+ w = 1+ elif rmse_type == 3:+ w = torch.abs(data)+ else:+ w = torch.sqrt(torch.abs(data))+ sumlx = torch.sum(w * data * L, axis=-1)+ suml2 = torch.sum(w * L * L, axis=-1)+ scales = torch.where(suml2 !=0, sumlx / suml2, 0)+ if return_early:+ iscales_inv = torch.where(iscales != 0, 1 / iscales, 0).reshape(iscales.shape[:2])+ scales = torch.where(suml2 > 0, 0.5 * (scales + iscales_inv), iscales_inv)+ L = (L + nmax).to(torch.uint8)+ return scales, L+ best = scales * sumlx+ for _is in range(-9, 10):+ if _is == 0:+ continue+ iscales = torch.where(group_max != 0, -(nmax + -0.1 * _is) / nmax, 0)+ tmp_L = torch.round(iscales * data).clip(-nmax, nmax - 1)+ sumlx = torch.sum(w * data * L, axis=-1)+ suml2 = torch.sum(w * L * L, axis=-1)++ replace_id = (suml2 > 0) & (sumlx*sumlx > best*suml2)+ L[replace_id] = tmp_L[replace_id]+ scales[replace_id] = sumlx[replace_id] / suml2[replace_id]+ best[replace_id] = scales[replace_id] * sumlx[replace_id]++ L = (L + nmax).to(torch.uint8)+ return scales, L+++def make_q3_quants(data, bits, do_rmse=False):+ nmax = pow(2, bits - 1)+ imax = abs(data).argmax(axis=-1, keepdims=True)+ group_max = torch.take_along_dim(data, imax, axis=-1)+ iscale = torch.where(group_max != 0, -nmax / group_max, 0)+ if do_rmse:+ L = torch.round(iscale * data).clip(-nmax, nmax - 1)+ w = torch.pow(data, 2)+ sumlx = torch.sum(w * data * L, axis=-1)[0]+ suml2 = torch.sum(w * L * L, axis=-1)[0]++ for itry in range(5):+ # w = np.power(data, 2)+ if len(sumlx.shape) == 2:+ sumlx = sumlx.unsqueeze(-1)+ if len(suml2.shape) == 2:+ suml2 = suml2.unsqueeze(-1)+ slx = sumlx - w*data*L+ repalce_idx = slx > 0+ sl2 = suml2 - w*L*L+ new_L = torch.round(data * sl2 / slx).clip(-nmax, nmax - 1)+ tmp_repalce_idx = repalce_idx & (new_L != L)+ slx[tmp_repalce_idx] += w[tmp_repalce_idx] * data[tmp_repalce_idx] * new_L[tmp_repalce_idx]+ sl2[tmp_repalce_idx] += w[tmp_repalce_idx] * new_L[tmp_repalce_idx] * new_L[tmp_repalce_idx]+ repalce_idx &= (sl2 > 0) & (slx * slx * suml2 > sumlx * sumlx * sl2)++ L[repalce_idx] = new_L[repalce_idx]+ sumlx = slx+ suml2 = sl2++ L = torch.round(iscale * data).clip(-nmax, nmax - 1) + nmax+ scales = torch.where(iscale != 0, 1 / iscale, 0).reshape(iscale.shape[:2])+ return scales, L.to(torch.uint8)+++def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):+ nmax = pow(2, bits) - 1+ # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K+ if len(data.shape) == 2:+ if bits in [4, 5]:+ data_shape = (-1, 8, 32)+ elif bits in [2, 3]:+ data_shape = (-1, 16, 16)+ else:+ raise NotImplementedError(f"bits = {bits} is not supported")+ data = data.reshape(data_shape)+ sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)+ av_x = torch.sqrt(sum_x2 / 32)+ weight = torch.abs(data) + av_x++ group_min = torch.min(data, axis=-1, keepdims=True)[0]+ group_max = torch.max(data, axis=-1, keepdims=True)[0]++ the_mins = -group_min++ sum_w = torch.sum(weight, axis=-1, keepdims=True)+ sum_x = torch.sum(weight * data, axis=-1, keepdims=True)++ group_min[group_min > 0] = 0++ scale = (group_max-group_min) / nmax+ iscale = torch.where(scale == 0, 0, 1 / scale)++ l_values = torch.round(iscale * (data-group_min))+ L = torch.clip(l_values, 0, nmax).to(torch.uint8)++ diffs = scale*L + group_min - data+ diffs = torch.abs(diffs) if use_mad else diffs**2+ best_mad = torch.sum(weight * diffs, axis=-1, keepdims=True)++ if nstep < 1:+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])++ for step in range(nstep):+ new_scale = (group_max-group_min) / (rmin + rdelta*step + nmax)+ new_iscale = torch.where(new_scale == 0, 0, 1 / new_scale)++ l_values = torch.round(new_iscale * (data-group_min))+ Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)++ sum_l = torch.sum(weight * Laux, axis=-1, keepdims=True)+ sum_l2 = torch.sum(weight * Laux**2, axis=-1, keepdims=True)+ sum_xl = torch.sum(weight * Laux * data, axis=-1, keepdims=True)++ D = sum_w*sum_l2 - sum_l*sum_l+ replace_idx = D > 0++ this_scale = (sum_w*sum_xl - sum_x*sum_l) / D+ this_min = (sum_l2*sum_x - sum_l*sum_xl) / D+ this_min[this_min > 0] = 0+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]++ diffs = this_scale*Laux + this_min - data+ diffs = torch.abs(diffs) if use_mad else diffs**2+ mad = torch.sum(weight * diffs, axis=-1, keepdims=True)++ replace_idx &= mad < best_mad+ best_mad[replace_idx] = mad[replace_idx]+ L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]+ scale[replace_idx] = this_scale[replace_idx]+ group_min[replace_idx] = this_min[replace_idx]++ the_mins = -group_min+ return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])+++@register_qtype("bf16")def bf16_quant_block(blocks, scale=None, zp=None, **kwargs):n = blocks.view(torch.uint32)# force nan to quiet@@ -79,12 +227,13 @@ def bf16_quant_block(blocks, scale=None, zp=None, **kwargs):return n.cpu().numpy().astype(np.uint16).view(np.uint8)-@register_block("q4_0")-def q4_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):+@register_qtype("q4_0")+def q4_0_quant_block(blocks, scale=None, zp=None, **kwargs):if scale is not None:d = scale.reshape((-1, 1))else:- max = torch.max(blocks, axis=-1, keepdims=True)[0]+ imax = abs(blocks).argmax(axis=-1, keepdims=True)+ max = np.take_along_axis(blocks, imax, axis=-1)d = max / -8id = torch.where(d == 0, 0, 1 / d)@@ -100,8 +249,8 @@ def q4_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):return np.concatenate([d, qs], axis=-1)-@register_block("q4_1")-def q4_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):+@register_qtype("q4_1")+def q4_1_quant_block(blocks, scale=None, zp=None, **kwargs):if scale is not None:d = scale.reshape((-1, 1))min = zp.reshape((-1, 1)) * d * -1@@ -123,107 +272,91 @@ def q4_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):return np.concatenate([d, m, qs], axis=-1)-@register_block("q8_0")-def q8_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs) -> np.ndarray:+@register_qtype("q5_0")+def q5_0_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):if scale is not None:d = scale.reshape((-1, 1))else:- d = torch.abs(blocks).max(axis=1, keepdims=True)[0] / 127+ imax = abs(blocks).argmax(axis=-1, keepdims=True)+ max = torch.take_along_dim(blocks, imax, axis=-1)[0]+ d = max / -16+id = torch.where(d == 0, 0, 1 / d)+ n_blocks = blocks.shape[0]+ block_size = GGML_QUANT_SIZES["q5_0"][0]- qs = torch_roundf(blocks * id)+ # FIXME: Q5_0's reference rounding is cursed and depends on FMA+ q = torch.trunc(blocks.to(torch.float64) * id.to(torch.float64) + 16.5).to(torch.uint8).clip(0, 31).cpu().numpy()- # (n_blocks, 2)- d = d.cpu().numpy().astype(np.float16).view(np.uint8)- # (n_blocks, block_size)- qs = qs.cpu().numpy().astype(np.int8).view(np.uint8)-- return np.concatenate([d, qs], axis=1)+ qs = q.reshape((n_blocks, 2, block_size // 2))+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)-def make_qkx2_quants(data, bits, rmin=-1, rdelta=0.1, nstep=20, use_mad=False):- assert bits in [2, 4], f"bits={bits} is not supported"- nmax = bits ** 2 -1- # data shape (nb, 8, 32) for Q4_K, (nb, 16, 16) for Q2_K- if len(data.shape) == 2:- data_shape = (-1, 8, 32) if bits == 4 else (-1, 16, 16)- data = data.reshape(data_shape)- sum_x2 = torch.sum(torch.pow(data, 2), axis=-1, keepdims=True)- av_x = torch.sqrt(sum_x2 / 32)- weight = torch.abs(data) + av_x-- group_min = torch.min(data, axis=-1, keepdims=True)[0]- group_max = torch.max(data, axis=-1, keepdims=True)[0]-- the_mins = -group_min+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)- sum_w = torch.sum(weight, axis=-1, keepdims=True)- sum_x = torch.sum(weight * data, axis=-1, keepdims=True)+ return np.concatenate([d, qh, qs], axis=-1)- group_min[group_min > 0] = 0- scale = (group_max - group_min) / nmax- iscale = torch.where(scale == 0, 0, 1 /scale)+@register_qtype("q5_1")+def q5_1_quant_block(blocks: np.array, scale=None, zp=None, **kwargs):+ if scale is not None:+ d = scale.reshape((-1, 1))+ min = zp.reshape((-1, 1)) * d * -1+ else:+ max = blocks.max(axis=-1, keepdims=True)[0]+ min = blocks.min(axis=-1, keepdims=True)[0]+ d = (max-min) / 31- l_values = torch.round(iscale * (data - group_min))- L = torch.clip(l_values, 0, nmax).to(torch.uint8)+ n_blocks = blocks.shape[0]+ block_size = GGML_QUANT_SIZES["q5_1"][0]- diffs = scale * L + group_min - data- diffs = torch.abs(diffs) if use_mad else diffs**2- best_mad = torch.sum(weight * diffs, axis=-1, keepdims=True)+ id = torch.where(d == 0, 0, 1 / d)+ q = torch.trunc((blocks-min) * id + 0.5).to(torch.uint8).clip(0, 31).cpu().numpy()- if nstep < 1:- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])+ qs = q.reshape((n_blocks, 2, block_size // 2))+ qs = (qs[..., 0, :] & np.uint8(0x0F)) | (qs[..., 1, :] << np.uint8(4))- for step in range(nstep):- new_scale = (group_max - group_min) / (rmin + rdelta * step + nmax)- new_iscale = torch.where(new_scale == 0, 0, 1 /new_scale)+ qh = np.packbits(q.reshape((n_blocks, 1, 32)) >> np.uint8(4), axis=-1, bitorder="little").reshape(n_blocks, 4)- l_values = torch.round(new_iscale * (data - group_min))- Laux = torch.clip(l_values, 0, nmax).to(torch.uint8)+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)+ m = min.cpu().numpy().astype(np.float16).view(np.uint8)- sum_l = torch.sum(weight * Laux, axis=-1, keepdims=True)- sum_l2 = torch.sum(weight * Laux**2, axis=-1, keepdims=True)- sum_xl = torch.sum(weight * Laux * data, axis=-1, keepdims=True)+ return np.concatenate([d, m, qh, qs], axis=-1)- D = sum_w * sum_l2 - sum_l * sum_l- replace_idx = D > 0- this_scale = (sum_w * sum_xl - sum_x * sum_l) / D- this_min = (sum_l2 * sum_x - sum_l * sum_xl) / D- this_min[this_min > 0] = 0- this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]+@register_qtype("q8_0")+def q8_0_quant_block(blocks, scale=None, zp=None, **kwargs) -> np.ndarray:+ if scale is not None:+ d = scale.reshape((-1, 1))+ else:+ d = torch.abs(blocks).max(axis=1, keepdims=True)[0] / 127+ id = torch.where(d == 0, 0, 1 / d)- diffs = this_scale * Laux + this_min - data- diffs = torch.abs(diffs) if use_mad else diffs**2- mad = torch.sum(weight * diffs, axis=-1, keepdims=True)+ qs = torch_roundf(blocks * id)- replace_idx &= mad < best_mad- best_mad[replace_idx] = mad[replace_idx]- L[replace_idx.reshape(replace_idx.shape[:2])] = Laux[replace_idx.reshape(replace_idx.shape[:2])]- scale[replace_idx] = this_scale[replace_idx]- group_min[replace_idx] = this_min[replace_idx]+ # (n_blocks, 2)+ d = d.cpu().numpy().astype(np.float16).view(np.uint8)+ # (n_blocks, block_size)+ qs = qs.cpu().numpy().astype(np.int8).view(np.uint8)- the_mins = -group_min- return scale.reshape(scale.shape[:2]), L, the_mins.reshape(the_mins.shape[:2])+ return np.concatenate([d, qs], axis=1)-@register_block("q2_k")-def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+@register_qtype("q2_k")+def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):nb = blocks.shape[0]output_scale = np.empty((nb, 16), dtype=np.uint8)output_qs = np.empty((nb, QK_K // 16 // 4, 16), dtype=np.uint8)- blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)+ blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)if scale is not None:scales = scale.reshape((-1, QK_K // 16))mins = wmin_m.reshape((-1, QK_K // 16))- output_d = d_scale.to(torch.float32) if len(d_scale.shape) == 2 \- else d_scale.reshape(-1, 1).to(torch.float32)- output_dmin = d_wmin_m.to(torch.float32) if len(d_wmin_m.shape) == 2 \- else d_wmin_m.reshape(-1, 1).to(torch.float32)- inv_scales = torch.where(d_scale == 0, 0, 1 / output_d)+ output_d = d_scale.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)+ inv_scales = torch.where(output_d == 0, 0, 1 / output_d)inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)@@ -245,19 +378,18 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleif d_scale is None:output_d = torch.where(max_scales > 0, max_scales / 15, 0)if d_wmin_m is None:- output_dmin = torch.where(max_mins > 0 ,max_mins / 15., 0)+ output_dmin = torch.where(max_mins > 0, max_mins / 15., 0)d_tmp = output_d * (output_scale & 0xF)dm_tmp = output_dmin * (output_scale >> 4)replace_ids = d_tmp != 0all_L[replace_ids] = torch.round(- (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(- torch.uint8)+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 3)output_scale = output_scale.cpu().numpy()- output_qs = all_L[:,::4] | (all_L[:,1::4] << 2) | (all_L[:,2::4] << 4) | (all_L[:,3::4] << 6)+ output_qs = all_L[:, ::4] | (all_L[:, 1::4] << 2) | (all_L[:, 2::4] << 4) | (all_L[:, 3::4] << 6)output_d = output_d.cpu().numpy()output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)output_dmin = output_dmin.cpu().numpy()@@ -268,7 +400,54 @@ def q2_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scalereturn np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)-@register_block("q4_k")+@register_qtype("q3_k")+def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ blocks = blocks.reshape(nb, QK_K // 16, 16)++ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)++ if scale is not None:+ scales = scale.reshape(-1, QK_K // 16)+ output_d = d_scale.reshape(-1, 1).to(torch.float32)+ iscales = torch.where(output_d == 0, 0, 1 / output_d)+ all_L = torch.round(blocks / scales.unsqueeze(-1)).clip(-4, 3) + 4+ all_L = all_L.to(torch.uint8)+ else:+ scales, all_L = make_q3_quants(blocks, bits=3, do_rmse=True)+ imax = abs(scales).argmax(axis=-1, keepdims=True)+ max_scales = torch.take_along_dim(scales, imax, axis=-1)+ iscales = torch.where(max_scales != 0, -32 / max_scales, 0)+ output_d = torch.where(iscales != 0, 1 / iscales, 0)++ q_scales = torch.round(iscales * scales).clip(-32, 31) + 32+ q_scales = q_scales.cpu().numpy().astype(np.uint8)+ output_scale[:, :8] = (q_scales[:, :8] & 0xF) | ((q_scales[:, 8:] & 0xF) << 4)+ hmask = q_scales >> 4+ output_scale[:, 8:] = hmask[:, :4] | hmask[:, 4:8] << 2 | hmask[:, 8:12] << 4 | hmask[:, 12:] << 6++ sc = torch.round(iscales * scales).clip(-32, 31)+ d = output_d.to(torch.float32) * sc+ replace_idx = (d != 0)++ all_L[replace_idx] = (torch.round(blocks[replace_idx] / d[replace_idx].unsqueeze(-1)).clip(-4, 3) + 4).to(+ torch.uint8)++ all_L = all_L.cpu().numpy()+ output_hmask = np.bitwise_or.reduce(+ all_L.reshape(nb, 8, 32) >> 2 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1) # pylint: disable=E1121+ all_L = np.where(all_L > 3, all_L - 4, all_L)++ output_qs = np.bitwise_or.reduce(+ all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121++ output_qs = output_qs.reshape(nb, 64).astype(np.uint8)+ output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)+ # [hmask, qs, scale, d]+ return np.concatenate([output_hmask, output_qs, output_scale, output_d], axis=-1)+++@register_qtype("q4_k")def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 32, 32))@@ -279,11 +458,9 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wif scale is not None:scales = scale.reshape(-1, QK_K // 32)mins = wmin_m.reshape(-1, QK_K // 32)- output_d = d_scale.to(torch.float32) if len(d_scale.shape) == 2 \- else d_scale.reshape(-1, 1).to(torch.float32)- output_dmin = d_wmin_m.to(torch.float32) if len(d_wmin_m.shape) == 2 \- else d_wmin_m.reshape(-1, 1).to(torch.float32)- inv_scales = torch.where(d_scale == 0, 0, 1 / output_d)+ output_d = d_scale.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)+ inv_scales = torch.where(output_d == 0, 0, 1 / output_d)inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)@@ -308,18 +485,18 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wq_scales = q_scales.cpu().numpy().astype(np.uint8)q_mins = q_mins.cpu().numpy().astype(np.uint8)- output_scale[:,:4] = q_scales[:,:4]- output_scale[:,4:8] = q_mins[:,:4]+ output_scale[:, :4] = q_scales[:, :4]+ output_scale[:, 4:8] = q_mins[:, :4]- output_scale[:,8:] = (q_scales[:,4:] & 0xF) | ((q_mins[:,4:] & 0xF) << 4)- output_scale[:,:4] |= ((q_scales[:,4:] >> 4) << 6)- output_scale[:,4:8] |= ((q_mins[:,4:] >> 4) << 6)+ output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)+ output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)+ output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)replace_ids = d_tmp != 0all_L[replace_ids] = torch.round((blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 15)- output_qs = all_L[:,::2] | (all_L[:,1::2] << 4)+ output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)output_d = output_d.cpu().numpy()output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)@@ -329,3 +506,105 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_w# [d, dmin, scale, qs]return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)+++@register_qtype("q5_k")+def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ blocks = blocks.reshape((nb, QK_K // 32, 32))++ output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)+ output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)+ output_qh = np.empty((nb, QK_K // 8), dtype=np.uint8)++ if scale is not None:+ scales = scale.reshape(-1, QK_K // 32)+ mins = wmin_m.reshape(-1, QK_K // 32)+ output_d = d_scale.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)+ inv_scales = torch.where(output_d == 0, 0, 1 / output_d)+ inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0] # (nb, 1)+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0] # (nb, 1)+ all_L = torch.round((blocks + mins.reshape(*mins.shape, 1)) / scales.reshape(*scales.shape, 1)).to(torch.uint8)+ else:+ scales, all_L, mins = make_qkx2_quants(blocks, bits=5, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=False)+ max_scales = torch.max(scales, axis=-1, keepdims=True)[0]+ max_mins = torch.max(mins, axis=-1, keepdims=True)[0]+ inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)+ inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)++ q_scales = torch.round(inv_scales * scales).to(torch.uint8).clip(0, 63)+ q_mins = torch.round(inv_mins * mins).to(torch.uint8).clip(0, 63)++ if d_scale is None:+ output_d = max_scales / 63+ if d_wmin_m is None:+ output_dmin = max_mins / 63++ d_tmp = output_d * q_scales+ dm_tmp = output_dmin * q_mins++ q_scales = q_scales.cpu().numpy().astype(np.uint8)+ q_mins = q_mins.cpu().numpy().astype(np.uint8)+ output_scale[:, :4] = q_scales[:, :4]+ output_scale[:, 4:8] = q_mins[:, :4]++ output_scale[:, 8:] = (q_scales[:, 4:] & 0xF) | ((q_mins[:, 4:] & 0xF) << 4)+ output_scale[:, :4] |= ((q_scales[:, 4:] >> 4) << 6)+ output_scale[:, 4:8] |= ((q_mins[:, 4:] >> 4) << 6)++ replace_ids = d_tmp != 0+ all_L[replace_ids] = torch.round(+ (blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).to(torch.uint8)+ all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 31) # (nb, 8, 32)++ output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)+ output_qh = np.bitwise_or.reduce(+ all_L >> 4 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1).astype(np.uint8) # pylint: disable=E1121++ output_d = output_d.cpu().numpy()+ output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_dmin = output_dmin.cpu().numpy()+ output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)+ output_qs = output_qs.reshape(nb, QK_K // 2)+ # [d, dmin, scale, qh, qs]+ return np.concatenate([output_d, output_dmin, output_scale, output_qh, output_qs], axis=-1)+++@register_qtype("q6_k")+def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None):+ nb = blocks.shape[0]+ blocks = blocks.reshape((nb, QK_K // 16, 16))++ scale = None+ if scale is not None:+ scales = scale.reshape(-1, QK_K // 16)+ output_d = d_scale.reshape(-1, 1).to(torch.float32)+ iscales = torch.where(output_d == 0, 0, 1 / output_d)+ all_L = (torch.round(blocks / scales.reshape(*scales.shape, 1)).clip(-32, 31) + 32).to(torch.uint8)+ else:+ scales, all_L = make_qx_quant(blocks, bits=6, rmse_type=1, qw=None)+ imax = abs(scales).argmax(axis=-1, keepdims=True)+ max_scales = torch.take_along_dim(scales, imax, axis=-1)+ iscales = torch.where(max_scales != 0, -128 / max_scales, 0)+ output_d = torch.where(iscales != 0, 1 / iscales, -1)++ output_scale = torch.round(iscales * scales).clip(max=127).to(torch.int8)+ d_tmp = output_d * output_scale.to(torch.float32)+ replace_ids = d_tmp != 0+ all_L[replace_ids] = torch.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \+ .clip(0, 63).to(torch.uint8)++ all_L = all_L.cpu().numpy()+ tmp_L = all_L.reshape(nb, 4, 64) & 0xF+ output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2)+ output_qh = np.bitwise_or.reduce(+ (all_L >> 4).reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), # pylint: disable=E1121+ axis=2).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121++ output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)++ # [ql, qh, scale, d]+ output_scale = output_scale.cpu().numpy().view(np.uint8)+ return np.concatenate([output_ql, output_qh, output_scale, output_d], axis=-1)\ No newline at end of filediff --git a/auto_round/export/export_to_gguf/utils.py b/auto_round/export/export_to_gguf/utils.pynew file mode 100644index 000000000..99ab13c7d--- /dev/null+++ b/auto_round/export/export_to_gguf/utils.py@@ -0,0 +1,30 @@+# Copyright (c) 2025 Intel Corporation+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++QK_K = 256+K_SCALE_SIZE = 12+GGML_QUANT_SIZES = {+ "bf16": (1, 2),+ "q4_0": (32, 2 + 16),+ "q4_1": (32, 2 + 2 + 16),+ "q5_0": (32, 2 + 4 + 16),+ "q5_1": (32, 2 + 2 + 4 + 16),+ "q8_0": (32, 2 + 32),+ "q2_k": (256, 2 + 2 + QK_K//16 + QK_K//4),+ "q3_k": (256, 2 + QK_K // 4 + QK_K // 8 + 12),+ "q4_k": (256, 2 + 2 + QK_K//2 + 12),+ "q5_k": (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12),+ "q6_k": (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16),+ "q8_k": (256, 4 + QK_K + QK_K // 8)+}\ No newline at end of filediff --git a/auto_round/export/export_to_itrex/export.py b/auto_round/export/export_to_itrex/export.pyindex b5c508677..435eed66d 100644--- a/auto_round/export/export_to_itrex/export.py+++ b/auto_round/export/export_to_itrex/export.py@@ -225,13 +225,13 @@ def pack_model(else:scale = scale.to(dtype=convert_dtype)zp = zp.to(dtype=torch.int32)- if isinstance(m, transformers.modeling_utils.Conv1D):+ if isinstance(m, transformers.pytorch_utils.Conv1D):fp_weight = fp_weight.t_().contiguous()int_weight = quant_weight_w_scale(fp_weight, scale, zp, group_size, fp_weight.device)if isinstance(m, torch.nn.Linear):in_features = m.in_featuresout_features = m.out_features- elif isinstance(m, transformers.modeling_utils.Conv1D):+ elif isinstance(m, transformers.pytorch_utils.Conv1D):in_features = m.weight.shape[0]out_features = m.weight.shape[1]int_weight = int_weight.type(torch.int32)diff --git a/auto_round/script/llm.py b/auto_round/script/llm.pyindex 5a9594f61..f6046c64f 100644--- a/auto_round/script/llm.py+++ b/auto_round/script/llm.py@@ -35,6 +35,7 @@ from auto_round.utils import (clear_memory,get_device_and_parallelism,get_model_dtype,+ str2bool,set_cuda_visible_devices)os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"@@ -199,7 +200,6 @@ class BasicArgumentParser(argparse.ArgumentParser):type=str,help="the torch_dytpe to load the model for evaluation.")-class EvalArgumentParser(argparse.ArgumentParser):def __init__(self, *args, **kwargs):@@ -338,19 +338,18 @@ def tune(args):from transformers import AutoModelForCausalLM, AutoTokenizer, AutoModel, AutoConfigfrom auto_round.utils import detect_device, get_library_version- from auto_round.utils import logger, _gguf_args_check+ from auto_round.utils import loggerif args.format is None:args.format = "auto_round"+formats = args.format.lower().replace(' ', '').split(",")from auto_round.utils import supported_formatsfor format in formats:if format not in supported_formats:raise ValueError(f"{format} is not supported, we only support {supported_formats}")- args = _gguf_args_check(args)-if "auto_gptq" in args.format and args.asym is True:logger.warning("the auto_gptq kernel has issues with asymmetric quantization. ""It is recommended to use sym quantization or --format='auto_round'")@@ -411,7 +410,7 @@ def tune(args):layer_config = {}for n, m in model.named_modules():- if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.pytorch_utils.Conv1D):if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:layer_config[n] = {"bits": 16}logger.info(@@ -539,7 +538,7 @@ def tune(args):for file in os.listdir(eval_folder):gguf_file = file- logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")if eval_model_dtype == "float32" or eval_model_dtype == "auto":logger.warning("set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"@@ -642,7 +641,7 @@ def eval(args):from lm_eval.utils import make_table # pylint: disable=E0401tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)- logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")if eval_model_dtype == "float32" or eval_model_dtype == "auto":logger.warning("set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"@@ -706,7 +705,7 @@ def eval_task_by_task(eval_model_dtype = get_model_dtype(eval_model_dtype)if is_gguf_file:tokenizer = AutoTokenizer.from_pretrained(model, gguf_file=gguf_file)- logger.warning("evaluate gguf model is an experimental feature, the accuracy may not be accurate.")+ logger.warning("evaluate gguf model is an experimental feature, the accuracy may be not correct.")if eval_model_dtype == "float32" or eval_model_dtype == "auto":logger.warning("set '--eval_model_dtype bf16' can significantly speed up evaluation for gguf model,"diff --git a/auto_round/script/mllm.py b/auto_round/script/mllm.pyindex 7fc42d1ae..212fb902a 100644--- a/auto_round/script/mllm.py+++ b/auto_round/script/mllm.py@@ -23,7 +23,6 @@ from auto_round.utils import (get_device_and_parallelism,set_cuda_visible_devices,logger,- _gguf_args_check)@@ -299,9 +298,6 @@ def tune(args):if format not in supported_formats:raise ValueError(f"{format} is not supported, we only support {supported_formats}")- args = _gguf_args_check(args)--##must set this before import torchset_cuda_visible_devices(args.device)device_str, use_auto_mapping = get_device_and_parallelism(args.device)@@ -350,7 +346,7 @@ def tune(args):if args.fp_layers != "":fp_layers = args.fp_layers.replace(" ", "").split(",")for n, m in model.named_modules():- if not isinstance(m, (torch.nn.Linear, transformers.modeling_utils.Conv1D)):+ if not isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):continuefor fp_layer in fp_layers:if fp_layer in n:@@ -363,7 +359,7 @@ def tune(args):logger.warning(f"mixed precision exporting does not support {format} currently")for n, m in model.named_modules():- if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.modeling_utils.Conv1D):+ if isinstance(m, torch.nn.Linear) or isinstance(m, transformers.pytorch_utils.Conv1D):if m.weight.shape[0] % 32 != 0 or m.weight.shape[1] % 32 != 0:layer_config[n] = {"bits": 32}logger.info(diff --git a/auto_round/utils.py b/auto_round/utils.pyindex cbe7ae22d..e61ba480a 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -33,14 +33,26 @@ from auto_round.export.export_to_gguf.config import GGUF_CONFIGshared_cache_keys = ("position_ids", "cache_position", "position_embeddings")-supported_formats = (- "auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:gptqmodel", "auto_round:auto_awq",- "itrex", "itrex_xpu", "fake"-)+class SupportedFormats:+ def __init__(self):+ self._support_format = (+ "auto_round", "auto_gptq", "auto_awq", "auto_round:auto_gptq", "auto_round:gptqmodel",+ "auto_round:auto_awq", "itrex", "itrex_xpu", "fake")+ self._gguf_format = tuple(GGUF_CONFIG.keys())+ self._support_list = self._support_format + self._gguf_format++ def __contains__(self, key):+ return True if key in self._support_list else False-supported_formats = supported_formats + tuple(GGUF_CONFIG.keys())+ def __str__(self):+ return "(%s)" % ', '.join(self._support_format + ("gguf:q*_0", "gguf:q*_1", "gguf:q*_k_s"))++ def __getitem__(self, key):+ return self._support_list[key]-supported_layer_types = (torch.nn.Linear, transformers.modeling_utils.Conv1D)+supported_formats = SupportedFormats()++supported_layer_types = (torch.nn.Linear, transformers.pytorch_utils.Conv1D)supported_dtypes = ("int", "mx_fp", "fp", "nv_fp")@@ -744,7 +756,7 @@ def check_memory_availability(device, inputs, weight, org_seqlen, org_bs):def get_layer_names_in_block(model, supported_types=(torch.nn.Linear,- transformers.modeling_utils.Conv1D), quant_block_list=None):+ transformers.pytorch_utils.Conv1D), quant_block_list=None):"""Retrieves the names of layers within each block of the model.Returns:@@ -1038,7 +1050,7 @@ def get_fp_layer_names(model, fp_layers):fp_layers = fp_layers.replace(" ", "").split(",")all_layer_names = []for n, m in model.named_modules():- if isinstance(m, (torch.nn.Linear, transformers.modeling_utils.Conv1D)):+ if isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):all_layer_names.append(n)not_to_quantized_layers = []@@ -1076,7 +1088,7 @@ def check_awq_gemm_compatibility(model, bits, group_size, sym, layer_configs=Nonif bits != 4:return False, f"AutoAWQ GEMM kernel only supports 4 bits"for n, m in model.named_modules():- if isinstance(m, transformers.modeling_utils.Conv1D):+ if isinstance(m, transformers.pytorch_utils.Conv1D):return False, "AutoAWQ GEMM kernel does not support conv1d"layer_names = get_layer_names_in_block(model)@@ -1151,16 +1163,28 @@ def get_layer_features(layer):return None, None # Unsupported layer type-def _gguf_args_check(args):+def _gguf_args_check(args_or_ar, format_str=None):from auto_round.utils import loggerfrom auto_round.export.export_to_gguf.config import GGUF_CONFIG+ import argparse- formats = args.format.lower().replace(' ', '').split(",")+ if format_str is None:+ args_or_ar.format = args_or_ar.format.replace("q*_", f"q{args_or_ar.bits}_")+ format_str = args_or_ar.format+ else:+ format_str = format_str.replace("q*_", f"q{args_or_ar.bits}_")+ formats = format_str.lower().replace(' ', '').split(",")formats = sorted(formats, key=lambda x: len(x))+ for f in formats:+ if f.startswith("gguf") and f not in GGUF_CONFIG:+ logger.error(f"{f} is not supported, please check.")pattern = re.compile("q\d_k")pre_dq_format = ""+ unsupport_list, reset_list = [], []for format in GGUF_CONFIG:if format in formats:+ if format == "q6_k_s":+ logger.warning("Please not that q6_k_s is q6_k.")try:from auto_round.export.export_to_gguf.convert import Modelexcept:@@ -1175,10 +1199,10 @@ def _gguf_args_check(args):else:pre_dq_format = format- if os.path.isdir(args.model):+ if isinstance(args_or_ar.model, str) and os.path.isdir(args_or_ar.model):from pathlib import Pathfrom auto_round.export.export_to_gguf.convert import Model- hparams = Model.load_hparams(Path(args.model))+ hparams = Model.load_hparams(Path(args_or_ar.model))model_architecture = hparams["architectures"][0]try:model_class = Model.from_model_architecture(model_architecture)@@ -1187,7 +1211,7 @@ def _gguf_args_check(args):sys.exit(1)if re.search(pattern, format) and ("hidden_size" in hparams and hparams["hidden_size"] % 256 != 0):- model_name = args.model.split('/')+ model_name = args_or_ar.model.split('/')model_name = model_name[-1] if model_name[-1] else model_name[-2]hidden_size = hparams["hidden_size"]logger.error(@@ -1198,17 +1222,28 @@ def _gguf_args_check(args):unsupport_list, reset_list = [], []gguf_config = GGUF_CONFIG[format]for k, v in gguf_config.items():- if getattr(args, k) != v:- unsupport_list.append(f"{k}={getattr(args, k)}")+ if not hasattr(args_or_ar, k):+ continue+ if k == "data_type":+ if re.search("q\d_1", format) and len(formats) > 1:+ v = "int"+ if getattr(args_or_ar, k) != v:+ unsupport_list.append(f"{k}={getattr(args_or_ar, k)}")reset_list.append(f"{k}={v}")- setattr(args, k, v)+ setattr(args_or_ar, k, v)if len(unsupport_list) > 0:logger.error(f"format {format} does not support for {', '.join(unsupport_list)},"f" reset to {', '.join(reset_list)}.")- logger.info(f"export format {format}, sym = {not args.asym}, group_size = {args.group_size}")-- return args+ if not isinstance(args_or_ar, argparse.Namespace) and len(unsupport_list) > 0:+ for layer_name in args_or_ar.layer_config:+ if args_or_ar.layer_config[layer_name]['bits'] >= 16:+ continue+ for k in args_or_ar.layer_config[layer_name]:+ if hasattr(args_or_ar, k):+ args_or_ar.layer_config[layer_name][k] = getattr(args_or_ar, k)+ args_or_ar.has_qlayer_outside_block = args_or_ar.set_layerwise_config(args_or_ar.layer_config)+ return args_or_ardef _to_model_dtype(model, model_dtype):@@ -1438,6 +1473,16 @@ def get_model_dtype(model_dtype, default="auto"):model_dtype = defaultreturn model_dtype+def str2bool(v):+ import argparse+ if isinstance(v, bool):+ return v+ if v.lower() in ('yes', 'true', 't', 'y', '1'):+ return True+ elif v.lower() in ('no', 'false', 'f', 'n', '0'):+ return False+ else:+ raise argparse.ArgumentTypeError('Boolean value expected.')def filter_quantization_config(quantization_config):default_dict = {"amp": True, "batch_size": 8, "data_type": int, "dataset": "NeelNanda/pile-10k",diff --git a/auto_round/wrapper.py b/auto_round/wrapper.pyindex 3d787c128..703c2b1f5 100644--- a/auto_round/wrapper.py+++ b/auto_round/wrapper.py@@ -93,7 +93,7 @@ class WrapperLinear(torch.nn.Module):orig_layer = self.orig_layerorig_weight = getattr(orig_layer, "get_weight", lambda: orig_layer.weight)()- if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):orig_weight = orig_weight.t()weight_reshape = reshape_and_pad_tensor(orig_weight.data, orig_layer.group_size)self.weight_min = torch.clamp(weight_reshape.min(1)[0], max=0)@@ -106,7 +106,6 @@ class WrapperLinear(torch.nn.Module):self.weight_quant_func, self.data_type = get_quant_func(orig_layer.data_type, orig_layer.bits,orig_layer.sym)-if self.enable_act_quant:self.act_quant_func, self.act_data_type = get_quant_func(orig_layer.act_data_type,orig_layer.act_bits,@@ -154,7 +153,7 @@ class WrapperLinear(torch.nn.Module):weight = self.orig_layer.weightif weight.device.type == 'meta':weight = self.orig_layer.get_weight().to(self.device)- if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):weight = weight.t()quant_kwargs = {}@@ -177,7 +176,7 @@ class WrapperLinear(torch.nn.Module):**quant_kwargs)weight_q = weight_q.to(weight.dtype)- if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):weight_q = weight_q.t()return weight_q, scale, zp@@ -237,7 +236,7 @@ class WrapperLinear(torch.nn.Module):self.orig_layer.weight.grad = Noneshape = qdq_weight.shape- if isinstance(self.orig_layer, transformers.modeling_utils.Conv1D):+ if isinstance(self.orig_layer, transformers.pytorch_utils.Conv1D):shape = qdq_weight.t().shapedef _set_dict_attr(attr_dict, attr_name):@@ -507,14 +506,17 @@ def wrapper_block(block, enable_minmax_tuning, enable_norm_bias_tuning, device='quantized_layers = []unquantized_layers = []for n, m in block.named_modules():- if isinstance(m, (torch.nn.Linear, transformers.modeling_utils.Conv1D)):+ if isinstance(m, (torch.nn.Linear, transformers.pytorch_utils.Conv1D)):if not check_to_quantized(m):unquantized_layers.append(n)continue- new_m = WrapperLinear(m, enable_minmax_tuning=enable_minmax_tuning,- enable_norm_bias_tuning=enable_norm_bias_tuning, device=device,- **kwargs,- )+ new_m = WrapperLinear(+ m,+ enable_minmax_tuning=enable_minmax_tuning,+ enable_norm_bias_tuning=enable_norm_bias_tuning,+ device=device,+ **kwargs,+ )set_module(block, n, new_m)quantized_layers.append(n)diff --git a/test/requirements.txt b/test/requirements.txtindex 7bf7c3dd3..bc6615906 100644--- a/test/requirements.txt+++ b/test/requirements.txt@@ -1,3 +1,4 @@addictmodelscopegguf+torchvisiondiff --git a/test/test_autoround.py b/test/test_autoround.pyindex 77ad0dacc..0fb7feed1 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -416,7 +416,10 @@ class TestAutoRound(unittest.TestCase):model_name, quantization_config=quantization_config, device_map="cpu", torch_dtype=torch.float16)self.assertTrue(isinstance(model.visual.blocks[0].attn.qkv, torch.nn.Linear))self.assertFalse(isinstance(model.visual.merger.mlp[0], QuantLinear))- self.assertTrue(isinstance(model.model.layers[0].self_attn.v_proj, QuantLinear))+ if hasattr(model.model, "language_model"):+ self.assertTrue(isinstance(model.model.language_model.layers[0].self_attn.v_proj, QuantLinear))+ else:+ self.assertTrue(isinstance(model.model.layers[0].self_attn.v_proj, QuantLinear))processor = AutoProcessor.from_pretrained(model_name, size=None)image_url = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/demo.jpeg"diff --git a/test/test_gguf_format.py b/test/test_gguf_format.pyindex c85eded58..d3ab8cb54 100644--- a/test/test_gguf_format.py+++ b/test/test_gguf_format.py@@ -62,10 +62,9 @@ class TestGGUF(unittest.TestCase):iters=1,data_type="int")- autoround.quantize()quantized_model_path = "./saved"- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_0")gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_0.gguf"model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"@@ -73,7 +72,7 @@ class TestGGUF(unittest.TestCase):print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))from auto_round.eval.evaluation import simple_evaluate_user_model- result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa")+ result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")# 0.246self.assertGreater(result['results']['openbookqa']['acc,none'], 0.23)shutil.rmtree("./saved", ignore_errors=True)@@ -89,10 +88,9 @@ class TestGGUF(unittest.TestCase):iters=1,data_type="int")- autoround.quantize()quantized_model_path = "./saved"- autoround.save_quantized(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q4_1")gguf_file = "Qwen2.5-0.5B-Instruct-494M-Q4_1.gguf"model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")text = "There is a girl who likes adventure,"@@ -100,10 +98,100 @@ class TestGGUF(unittest.TestCase):print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))from auto_round.eval.evaluation import simple_evaluate_user_model- result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa")+ result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="openbookqa", eval_model_dtype="bf16")# 0.23self.assertGreater(result['results']['openbookqa']['acc,none'], 0.22)shutil.rmtree("./saved", ignore_errors=True)++ def test_func(self):+ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=0,+ data_type="int"+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_1")+ self.assertTrue(autoround.group_size == 32)+ self.assertFalse(autoround.sym)+ gguf_file = os.listdir("saved")[0]+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")+ text = "There is a girl who likes adventure,"+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))+ shutil.rmtree("./saved", ignore_errors=True)++ model_name = "Qwen/Qwen2.5-1.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=3,+ group_size=16,+ sym=True,+ iters=0,+ data_type="int_sym_dq",+ super_group_size=16,+ super_bits=6+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")+ from auto_round.eval.evaluation import simple_evaluate_user_model+ gguf_file = os.listdir("saved")[0]+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")+ result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="lambada_openai", eval_model_dtype="bf16")+ self.assertGreater(result['results']['lambada_openai']['acc,none'], 0.5)+ shutil.rmtree("./saved", ignore_errors=True)++ def test_q5_k(self):+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=5,+ group_size=32,+ sym=False,+ iters=0,+ data_type="int_asym_dq",+ super_group_size=8,+ super_bits=6+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")+ gguf_file = os.listdir("saved")[0]+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")+ text = "There is a girl who likes adventure,"+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))+ shutil.rmtree("./saved", ignore_errors=True)++ def test_q6_k(self):+ model_name = "Qwen/Qwen2.5-1.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ autoround = AutoRound(+ model,+ self.tokenizer,+ bits=6,+ group_size=16,+ sym=True,+ iters=0,+ data_type="int_sym_dq",+ super_group_size=16,+ super_bits=8+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, inplace=False, format="gguf:q*_k_s")+ gguf_file = os.listdir("saved")[0]+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path, gguf_file=gguf_file, device_map="auto")+ text = "There is a girl who likes adventure,"+ inputs = self.tokenizer(text, return_tensors="pt").to(model.device)+ print(self.tokenizer.decode(model.generate(**inputs, max_new_tokens=10)[0]))+ shutil.rmtree("./saved", ignore_errors=True)if __name__ == "__main__":unittest.main()