PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf bug of moe models and lmhead/embedding bits setting regression (#628)
Heng Guo20eed5b6c469
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex fc3f74b80..25f832b6b 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -430,7 +430,7 @@ class AutoRound(object):if "mx_fp" in self.data_type and self.group_size != 32:logger.warning("mx_fp should only support group_size of 32 in real deployment")- if "nv_fp" in self.data_type and (self.group_size != 32 or self.group_size!=16):+ if "nv_fp" in self.data_type and (self.group_size != 32 or self.group_size != 16):logger.warning("mx_fp should only support group_size of 16/32 in real deployment")if self.nsamples < self.gradient_accumulate_steps * self.batch_size:@@ -466,7 +466,7 @@ class AutoRound(object):raise ValueError("gguf format is not compatible with other formats, please choose only one of them")if has_gguf and self.iters != 0 and self.bits != 3:logger.warning(- "We recommend setting `iters=0` when exporting to GGUF format,"+ "`iters=0` is recommended when exporting to GGUF format except for bits 3,"" as we have optimized the RTN method for this case."" We are likely to release new algorithm for certain configurations in the future.")@@ -498,8 +498,8 @@ class AutoRound(object):"You can also try to increase the model_max_length to avoid this issue.")self.seqlen = min(self.seqlen, self.tokenizer.model_max_length)- if self.group_size==0 and "fp8" not in self.data_type:- logger.warning("group_size of 0 is not supported for data_type other than fp8 ")+ if self.group_size == 0 and "fp8" not in self.data_type:+ logger.warning("group_size of 0 is not supported for data_type other than fp8 ")def quantize_and_save(self, output_dir: str = "tmp_autoround", format: str = "auto_round", inplace=True, **kwargs):"""Quantizes the model and saves it in the specified format(s).@@ -539,7 +539,7 @@ class AutoRound(object):if not ("gguf" in format_ or "fake" in format_):only_gguf = Falsebreak- if len(formats)==1 and "fake" == formats[0]:+ if len(formats) == 1 and "fake" == formats[0]:only_gguf = Falseif only_gguf:self.scale_dtype = torch.float32@@ -691,8 +691,8 @@ class AutoRound(object):if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:from accelerate.big_modeling import dispatch_modeldispatch_model(model, model.hf_device_map)- else:- model = self.model.to(self.device)++def register_act_hook(model):def get_act_max_hook(module, input, output):@@ -716,6 +716,7 @@ class AutoRound(object):hooks = register_act_hook(self.model)try:+ model = self.model.to(self.device)cnt = 0for data in self.dataloader:cnt += data["input_ids"].shape[0]@@ -724,9 +725,15 @@ class AutoRound(object):if cnt > self.nsamples:breakexcept RuntimeError as e:- if "CUDA out of memory" in str(e) or "MODULE:PT_DEVMEM" in str(e):- logger.warning("out of vram, fallback to cpu, pleaser use more gpus by setting `--device 0,1,2,3`")- model = self.model.to("cpu")+ try:+ model=self.model.to("cpu")+ clear_memory()+ logger.warning("out of vram, fallback to blockwise way to get imatrix, this may affect accuracy")+ self.model_forward_blockwise()+ except Exception as e:+ model=self.model.to("cpu")+ clear_memory()+ logger.warning("fallback to cpu, pleaser use more gpus to get imatrix, by setting `--device 0,1,2,3`")if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:accelerate.hooks.remove_hook_from_submodules(self.model) ##self.model.hf_device_map has not been changed@@ -737,8 +744,6 @@ class AutoRound(object):model(**data)if cnt > self.nsamples:break- else:- raisefor hook in hooks:hook.remove()@@ -776,7 +781,8 @@ class AutoRound(object):keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]self.layer_config[embedding_name] = self.layer_config.get(embedding_name, {})for key in keys:- if key in self.layer_config[embedding_name]:+ if key in self.layer_config[embedding_name]:+ self.layer_config["fixed_by_user"] = Truecontinueself.layer_config[embedding_name][key] = getattr(config, "get")(key)setattr(get_module(self.model, embedding_name), key, config.get(key))@@ -796,7 +802,8 @@ class AutoRound(object):keys = ["bits", "group_size", "super_bits", "super_group_size", "data_type", "sym"]self.layer_config[lm_head_name] = self.layer_config.get(lm_head_name, {})for key in keys:- if key in self.layer_config[lm_head_name]:+ if key in self.layer_config[lm_head_name] and self.layer_config[lm_head_name].get("fixed_by_user",+ False):continueself.layer_config[lm_head_name][key] = getattr(config, "get")(key)setattr(get_module(self.model, lm_head_name), key, config.get(key))@@ -866,15 +873,87 @@ class AutoRound(object):self.quantized = Truereturn self.model, self.layer_config- def quantize(self):- """Quantize the model and return the quantized model along with layer configurations.- the entry of AutoRound.+ def model_forward_blockwise(self):+ if bool(self.quant_block_list):+ all_blocks = self.quant_block_list+ else:+ all_blocks = get_block_names(self.model)+ if len(all_blocks) == 0:+ raise ValueError("could not find any blocks, please check the model or quant_block_list")++ ##ignore layers+ all_first_block_names = [block[0] for block in all_blocks]+ all_inputs = self.cache_inter_data(+ all_first_block_names, self.nsamples)+ if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:+ accelerate.hooks.remove_hook_from_submodules(self.model) ##self.model.hf_device_map has not been changed++ for block_names in all_blocks:+ inputs = all_inputs[block_names[0]]+ all_inputs.pop(block_names[0])+ keys = inputs.keys()+ input_id_str = [key for key in keys if key.startswith('hidden_state')]+ if len(input_id_str) != 1:+ raise RuntimeError(f"hidden_states arg mismatch error,"+ "please raise an issue in https://github.com/intel/auto-round/issues")+ inputs["input_ids"] = inputs.pop(input_id_str[0], None)++ clear_memory(self.inputs)++ if "input_ids" in inputs.keys():+ total_samples = len(inputs["input_ids"])+ if total_samples < self.batch_size:+ self.batch_size = total_samples+ logger.warning(f"force the train batch size to {total_samples}")++ input_ids = inputs["input_ids"]+ inputs.pop("input_ids", None)+ for block_name in block_names:+ input_others = inputs+ clear_memory()+ input_ids = to_device(input_ids, self.cache_device)+ input_others = to_device(input_others, self.cache_device)+ ## as in the calibration phase, we may use bf16 for calibration due to low_gpu_memory usage+ tmp_dtype = self.amp_dtype if self.amp else torch.float32+ for i in range(len(input_ids)):+ input_ids[i] = input_ids[i].to(tmp_dtype)++ for key in input_others.keys():+ if isinstance(input_others[key], torch.Tensor) and (+ input_others[key].dtype == torch.float16 or input_others[key].dtype == torch.bfloat16+ ):+ input_others[key] = input_others[key].to(tmp_dtype)+ elif isinstance(input_others[key], list):+ for i in range(len(input_others[key])):+ to_dtype(input_others[key][i], tmp_dtype)++ block = get_module(self.model, block_name)++ if self.device_map is not None:+ from accelerate import dispatch_model+ for n, m in block.named_modules():+ if len(list(m.children())) != 0 or not hasattr(m, "tuning_device"):+ continue+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module+ hook = AlignDevicesHook(m.tuning_device, io_same_device=True)+ add_hook_to_module(m, hook, True)++ input_ids = self.get_block_outputs(block, input_ids, input_others,+ self.batch_size * self.infer_bs_coeff,+ self.device,+ self.cache_device)++ if self.device_map is not None:+ accelerate.hooks.remove_hook_from_submodules(+ block)+ mv_module_from_gpu(block, self.low_cpu_mem_usage)++ def quantize(self):+ """Quantize the model and return the quantized model along with layer configurations.The entry of AutoRound.Returns:The quantized model and layer configurations."""- ## TODO add common check-self._check_compatibility()self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)if not hasattr(self, "formats"):@@ -1104,7 +1183,7 @@ class AutoRound(object):# Skip unsupported typessupported_types = self.supported_types- if not isinstance(m,supported_types):+ if not isinstance(m, supported_types):continueall_supported_layer_names.append(n)@@ -1122,8 +1201,8 @@ class AutoRound(object):for match_name in matched_names:layer_config[match_name] = valelse:- tmp_m = get_module(self.model,name)- if not isinstance(tmp_m, torch.nn.Embedding): ##TODO not good code style+ tmp_m = get_module(self.model, name)+ if not isinstance(tmp_m, torch.nn.Embedding): ##TODO not good code styleraise ValueError(f"key {name} in layer_config is invalid, please have a double check")has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)@@ -1145,6 +1224,7 @@ class AutoRound(object):for key in keys:if key not in layer_config[n].keys():layer_config[n][key] = getattr(self, key)+ layer_config[n]["fixed_by_user"] = True# If the layer is not in the config and not part of a quantization block,# use default configuration and set specific valueselse:@@ -1161,7 +1241,7 @@ class AutoRound(object):# If the layer is outside a block and requires quantization, mark it as a quantized layer outside the blockif (n not in layers_in_blocks and check_to_quantized(layer_config[n])- and not isinstance(m,torch.nn.Embedding)):+ and not isinstance(m, torch.nn.Embedding)):has_qlayer_outside_block = Truein_features, out_features = get_layer_features(m)@@ -1355,7 +1435,7 @@ class AutoRound(object):logger.info("switch to cpu to cache block inputs")if (self.has_qlayer_outside_block orself.__class__.__name__ == "AutoRoundMLLM"):- logger.warning("We strongly recommend using more GPUs in calibration."+ logger.warning("we strongly recommend using more GPUs in calibration."" Otherwise, some layers may fall back to `rtn` mode, which can affect accuracy.")if hasattr(self.model, "hf_device_map") and len(self.model.hf_device_map) > 1:accelerate.hooks.remove_hook_from_submodules(@@ -2072,7 +2152,7 @@ class AutoRound(object):save_quantized_as_format = EXPORT_FORMAT.get(format)if "gptq" in format and not self.sym:logger.warning(- "The asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"+ "the asymmetrical kernel of the GPTQ format may result in a noticeable accuracy drop,"" particularly for 2-bit quantization and smaller models."" We recommend exporting to either the AutoAWQ format ( only 4 bits) or ""the AutoRound format(2/3/4/8 bits)."diff --git a/auto_round/data_type/gguf.py b/auto_round/data_type/gguf.pyindex 0ab57d9b9..96d8bf7da 100644--- a/auto_round/data_type/gguf.py+++ b/auto_round/data_type/gguf.py@@ -139,7 +139,8 @@ def double_quant_tensor(tensor, bits):wmax = torch.clamp(tensor.max(-1)[0], min=0)scale = wmax / maxqscale = scale.view(-1, 1)- inverse_scale = torch.where(scale == 0, 0, 1 / scale)+ # inverse_scale = torch.where(scale == 0, 0, 1 / scale)+ inverse_scale = torch.where(wmax > 0, maxq / wmax, 0).view(-1, 1)qdq_tensor = torch.clamp(round_ste(tensor * inverse_scale), max=maxq) * scalereturn qdq_tensor, scale@@ -244,26 +245,26 @@ def quant_tensor_asym_dq(tensor, bits=4, group_size=-1, v=0, min_scale=1.0, max_scale = ((wmax - wmin) / maxq).to(scale_dtype)scale = torch.clamp(scale, min=q_scale_thresh)scale = scale.view(-1, super_group_size)- wmin_m = -wmin # pylint: disable=E1130- wmin_m = wmin_m.view(-1, super_group_size)+ wmin = -wmin # pylint: disable=E1130+ wmin = wmin.view(-1, super_group_size)##conduct double quantscale, d_scale = double_quant_tensor(scale, super_bits)- wmin_m, d_wmin_m = double_quant_tensor(wmin_m, super_bits)+ wmin, d_wmin = double_quant_tensor(wmin, super_bits)scale = scale.view(-1, 1)scale = torch.clamp(scale, q_scale_thresh)d_scale = torch.clamp(d_scale, q_scale_thresh)- d_wmin_m = torch.clamp(d_wmin_m, q_scale_thresh)- wmin_m = wmin_m.view(-1, 1)+ d_wmin = torch.clamp(d_wmin, q_scale_thresh)+ wmin = wmin.view(-1, 1)- int_w = round_ste((tensor + wmin_m) / scale + v)+ int_w = round_ste((tensor + wmin) / scale + v)q = torch.clamp(int_w, 0, maxq)- qdq_result = (scale * q - wmin_m).to(tensor.dtype)+ qdq_result = (scale * q - wmin).to(tensor.dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- # zp = round_ste(wmin_m / scale) # remove this later- return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}+ # zp = round_ste(wmin / scale) # remove this later+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}@register_dtype("rtn_int_asym_dq")@@ -324,18 +325,18 @@ def quant_tensor_gguf_asym_dq(av_x = torch.sqrt(sigma2)quant_weights = torch.abs(tensor) + av_xparams = search_kwargs[bits]- scale, wmin_m = iterative_wls_quant_search(+ scale, wmin = iterative_wls_quant_search(tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],use_mad=params["use_mad"], weights=quant_weights)scale = scale.to(scale_dtype)scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)scale = scale.reshape(-1, super_group_size)- wmin = wmin_m.reshape(-1, super_group_size)+ wmin = wmin.reshape(-1, super_group_size)scale, d_scale = double_quant_tensor(scale, super_bits)wmin = torch.where(torch.abs(wmin) < 1e-30, 0, wmin)- wmin_m, d_wmin_m = double_quant_tensor(wmin, super_bits)- wmin_m = wmin_m.view(-1, 1)+ wmin, d_wmin = double_quant_tensor(wmin, super_bits)+ wmin = wmin.view(-1, 1)scale = scale.view(-1, 1)else:imatrix = imatrix.to(tensor.device)@@ -359,7 +360,7 @@ def quant_tensor_gguf_asym_dq(params = search_kwargs[bits]- scale, wmin_m_0 = iterative_wls_quant_search(+ scale, wmin_0 = iterative_wls_quant_search(tensor, bits=bits, rrmin=params["rmin"], rdelta=params["rdelta"], nstep=params["nstep"],use_mad=params["use_mad"], weights=quant_weights)@@ -367,21 +368,22 @@ def quant_tensor_gguf_asym_dq(scale = torch.where(torch.abs(scale) < 1e-30, 0, scale)nmax = 2 ** super_bits - 1scale = scale.reshape(-1, super_group_size)- wmin_m = wmin_m_0.reshape(-1, super_group_size)+ wmin = wmin_0.reshape(-1, super_group_size)sum_quant_weights = quant_weights.sum(-1, keepdim=True).reshape(-1, super_group_size)d_scale, q_scale = make_qp_quants(nmax, scale, sum_quant_weights)- d_wmin_m, q_wmin_m = make_qp_quants(nmax, wmin_m, sum_quant_weights)+ d_wmin, q_wmin = make_qp_quants(nmax, wmin, sum_quant_weights)+d_scale = d_scale.unsqueeze(-1)- d_wmin_m = d_wmin_m.unsqueeze(-1)+ d_wmin = d_wmin.unsqueeze(-1)scale = (d_scale * q_scale).view(-1, 1)- wmin_m = (d_wmin_m * q_wmin_m).view(-1, 1)+ wmin = (d_wmin * q_wmin).view(-1, 1)inverse_scale = torch.where(scale == 0, 0, 1 / scale)- int_w = torch.clamp(round_ste((tensor + wmin_m) * inverse_scale + v), 0, maxq)- qdq_result = (scale * int_w - wmin_m).to(orig_dtype)+ int_w = torch.clamp(round_ste((tensor + wmin) * inverse_scale + v), 0, maxq)+ qdq_result = (scale * int_w - wmin).to(orig_dtype)qdq_result = revert_tensor_by_pad(qdq_result, orig_shape=orig_shape, pad_len=pad_len)- return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin_m": wmin_m, "d_wmin_m": d_wmin_m}+ return qdq_result, {"scale": scale, "d_scale": d_scale}, {"wmin": wmin, "d_wmin": d_wmin}def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, use_mad=False, weights=None):@@ -411,29 +413,38 @@ def iterative_wls_quant_search(data, bits=4, rrmin=-1.0, rdelta=0.1, nstep=20, usum_w = torch.sum(weights, dim=1, keepdim=True)sum_x = torch.sum(weights * data, dim=1, keepdim=True)- scale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))- quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)+ # scale = 1 / ((maxq - minq) / (rmax - rmin + 1e-8))+ scale = (rmax - rmin) / (maxq - minq)+ iscale = torch.where(scale == 0, 0, 1 / scale)+ # quant_data = torch.clamp(torch.round((maxq - minq) / (rmax - rmin + 1e-8) * (data - rmin)), minq, maxq)+ quant_data = torch.clamp(torch.round(iscale * (data - rmin)), minq, maxq)diff = scale * quant_data + rmin - databest_mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=1, keepdim=True)for is_ in range(nstep):factor = rrmin + rdelta * is_ + maxq - minq- iscale_new = factor / (rmax - rmin + 1e-8)+ # iscale_new = factor / (rmax - rmin + 1e-8)+ scale_new = (rmax - rmin) / factor+ iscale_new = torch.where(scale_new == 0, 0, 1 / scale_new)quant_data_new = torch.clamp(torch.round(iscale_new * (data - rmin)), minq, maxq)mul_weights_quant_data = weights * quant_data_new- sum_l = torch.sum(mul_weights_quant_data, dim=1, keepdim=True)- sum_l2 = torch.sum(mul_weights_quant_data * quant_data_new, dim=1, keepdim=True)- sum_xl = torch.sum(mul_weights_quant_data * data, dim=1, keepdim=True)+ sum_l = torch.sum(mul_weights_quant_data, dim=-1, keepdim=True)+ sum_l2 = torch.sum(mul_weights_quant_data * quant_data_new, dim=-1, keepdim=True)+ sum_xl = torch.sum(mul_weights_quant_data * data, dim=-1, keepdim=True)D = sum_w * sum_l2 - sum_l ** 2this_scale = (sum_w * sum_xl - sum_x * sum_l) / Dthis_min = (sum_l2 * sum_x - sum_l * sum_xl) / D+ this_min[this_min > 0] = 0+ this_scale[this_min > 0] = (sum_xl / sum_l2)[this_min > 0]+ reverse_this_scale = torch.where(this_scale == 0, 0, 1 / this_scale)- quant_data = torch.clamp(torch.round((1 / this_scale) * (data - this_min)), minq, maxq)+ quant_data = torch.clamp(torch.round(reverse_this_scale * (data - this_min)), minq, maxq)diff = this_scale * quant_data + this_min - data- mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=1, keepdim=True)+ # diff = this_scale * quant_data_new + this_min - data+ mad = torch.sum((weights * torch.abs(diff)) if use_mad else weights * diff ** 2, dim=-1, keepdim=True)idx_to_replace = torch.where((mad < best_mad) & (D > 0))[0]best_mad[idx_to_replace] = mad[idx_to_replace]diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex b0dbcd7b9..4a3b180c4 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -77,7 +77,6 @@ class SentencePieceTokenTypes(IntEnum):AnyModel = TypeVar("AnyModel", bound="type[Model]")-class OriModel:_model_classes: dict[str, type[Model]] = {}@@ -1130,6 +1129,30 @@ class Model(OriModel):# would require using decorated functions instead of simply defining the propertyif "model_arch" not in cls.__dict__:raise TypeError(f"Missing property 'model_arch' for {cls.__name__!r}")++ def get_moe_name(self, name, new_name):+ type_mapping = {+ "FFN_GATE_EXP": ["gate_proj", "w1", "linear"],+ "FFN_DOWN_EXP": ["down_proj", "w2", "linear_1"],+ "FFN_UP_EXP": ["up_proj", "w3", "linear_v"],+ }+ nums = re.findall('\d+', name)+ if len(nums) != 2:+ return name+ name_tmp = name[:-len(".weight")].replace(f'.{nums[1]}', '')+ new_name_tmp = new_name[:-len(".weight")]+ if self.tensor_map.get_type(name_tmp) == self.tensor_map.get_type(new_name_tmp):+ return name++ tensor_type = self.tensor_map.get_type(new_name_tmp).name+ experts_name = name_tmp.split(".")[-1]+ for k, v in type_mapping.items():+ if experts_name in v:+ idx = v.index(experts_name)+ name = name.replace(experts_name, type_mapping[tensor_type][idx])+ return name+ return name+def get_tensors(self) -> Iterator[tuple[str, Tensor]]:for name, tensor in self.model.named_parameters():@@ -1144,7 +1167,7 @@ class Model(OriModel):if hasattr(module, "scale"):if hasattr(self, "permute"):bs = module.weight.shape[0]- for attr in ["scale", "zp", "w_d_scale", "w_d_wmin_m", "w_wmin_m"]:+ for attr in ["scale", "zp", "w_d_scale", "w_d_wmin", "w_wmin"]:if hasattr(module, attr) and getattr(module, attr) is not None:attr_tensor = getattr(module, attr)ori_shape = attr_tensor.shape@@ -1164,17 +1187,17 @@ class Model(OriModel):zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zpif data_qtype.name.lower().endswith("_k"):d_scale = module.w_d_scale.to(torch.float32)- d_wmin_m = module.w_d_wmin_m.to(torch.float32) if hasattr(module, "w_d_wmin_m") else None- wmin_m = module.w_wmin_m.to(torch.float32) if hasattr(module, "w_wmin_m") else None+ d_wmin = module.w_d_wmin.to(torch.float32) if hasattr(module, "w_d_wmin") else None+ wmin = module.w_wmin.to(torch.float32) if hasattr(module, "w_wmin") else Nonedata = ggml_quant_gpu(data_torch,data_qtype.name.lower(),scale,zp,- wmin_m=wmin_m,+ wmin=wmin,d_scale=d_scale,- d_wmin_m=d_wmin_m,+ d_wmin=d_wmin,device=device)else:@@ -1199,6 +1222,8 @@ class Model(OriModel):def get_qtype_by_layer_config(self, layer_config, name, data_qtype):name = name[:-len('.weight')]if name not in layer_config or layer_config[name]['bits'] >= 16:+ if isinstance(data_qtype, bool):+ return gguf.GGMLQuantizationType.F32return data_qtypebits = layer_config[name]['bits']super_bits = layer_config[name]["super_bits"]@@ -1315,6 +1340,9 @@ class Model(OriModel):data_qtype = gguf.GGMLQuantizationType.F16# data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type+ # get name by new_name (for experts),+ name = self.get_moe_name(name, new_name)+# get data_qtype by layer_configdata_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)@@ -1384,6 +1412,8 @@ class Model(OriModel):shape_str = f"{{{', '.join(str(n) for n in reversed(shape))}}}"# n_dims is implicit in the shape+ if isinstance(data_qtype, bool):+ breakpoint()logger.info(f"{f'%-{max_name_len}s' % f'{new_name},'} {old_dtype}"f" --> {data_qtype.name}, shape = {shape_str}")diff --git a/auto_round/export/export_to_gguf/packing.py b/auto_round/export/export_to_gguf/packing.pyindex c5633e996..3fcc3f842 100644--- a/auto_round/export/export_to_gguf/packing.py+++ b/auto_round/export/export_to_gguf/packing.py@@ -28,22 +28,22 @@ def register_qtype(name):return register-def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, imatrix=None,+def ggml_quant_gpu(data, ggml_type, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, imatrix=None,device="cuda"):block_size, type_size = GGML_QUANT_SIZES[ggml_type]data = data.to(torch.float32).to(device)scale = scale.to(device) if scale is not None else scalezp = zp.to(device) if zp is not None else zp- wmin_m = wmin_m.to(device) if wmin_m is not None else wmin_m+ wmin = wmin.to(device) if wmin is not None else wmind_scale = d_scale.to(device) if d_scale is not None else d_scale- d_wmin_m = d_wmin_m.to(device) if d_wmin_m is not None else d_wmin_m+ d_wmin = d_wmin.to(device) if d_wmin is not None else d_wminshape = data.shapen_blocks = data.nelement() // block_sizeblocks = data.reshape((n_blocks, block_size))quant_func = GGML_QUANT_TYPE[ggml_type]- new_data = quant_func(blocks, scale, zp, wmin_m=wmin_m, d_scale=d_scale, d_wmin_m=d_wmin_m, imatrix=imatrix)+ new_data = quant_func(blocks, scale, zp, wmin=wmin, d_scale=d_scale, d_wmin=d_wmin, imatrix=imatrix)assert new_data.shape[-1] == type_sizenew_data = new_data.reshape(*shape[:-1], shape[-1] // block_size * type_size)@@ -404,28 +404,30 @@ def q8_0_quant_block(blocks, scale=None, zp=None, **kwargs) -> np.ndarray:@register_qtype("q2_k")-def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):+def q2_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 16, 16)) # (nb, 16, 16)if scale is not None:scales = scale.reshape((-1, QK_K // 16))- mins = wmin_m.reshape((-1, QK_K // 16))+ mins = wmin.reshape((-1, QK_K // 16))output_d = d_scale.reshape(-1, 1).to(torch.float32)- output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)inv_scales = torch.where(output_d == 0, 0, 1 / output_d)- inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)- max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)- max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)+ inv_mins = torch.where(d_wmin == 0, 0, 1 / output_dmin)+ max_scales = torch.max(scales, dim=-1, keepdim=True)[0]+ max_mins = torch.max(mins, dim=-1, keepdim=True)[0]all_L = torch.zeros_like(blocks, dtype=torch.uint8)else:scales, all_L, mins = make_qkx2_quants(blocks, bits=2, rmin=-0.5, rdelta=0.1, nstep=15, use_mad=True)max_scales = torch.max(scales, dim=-1, keepdim=True)[0]max_mins = torch.max(mins, dim=-1, keepdim=True)[0]+ output_d = torch.where(max_scales > 0, max_scales / 15, 0)+ output_dmin = torch.where(max_mins > 0, max_mins / 15., 0)inv_scales = torch.where(max_scales > 0, 15. / max_scales, 0)inv_mins = torch.where(max_mins > 0, 15. / max_mins, 0)-+replace_ids = (max_scales > 0).squeeze()output_scale = torch.zeros_like(scales).to(torch.uint8)output_scale[replace_ids] = torch.round(@@ -434,10 +436,6 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wreplace_ids = (max_mins > 0).squeeze()output_scale[replace_ids] |= torch.round(inv_mins[replace_ids] * mins[replace_ids]).clip(0, 15).to(torch.uint8) << 4- if d_scale is None:- output_d = torch.where(max_scales > 0, max_scales / 15, 0)- if d_wmin_m is None:- output_dmin = torch.where(max_mins > 0, max_mins / 15., 0)d_tmp = output_d * (output_scale & 0xF)dm_tmp = output_dmin * (output_scale >> 4)@@ -446,7 +444,6 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wall_L[replace_ids] = torch.round((blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 3).to(torch.uint8)- all_L = np.clip(all_L.cpu().numpy(), 0, 3).astype(np.uint8)output_scale = output_scale.cpu().numpy()all_L = all_L.reshape(-1, 4, 32)@@ -455,14 +452,13 @@ def q2_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_woutput_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)output_dmin = output_dmin.cpu().numpy()output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)- output_qs = output_qs.reshape((nb, QK_K // 4))+ output_qs = output_qs.cpu().numpy().astype(np.uint8).reshape((nb, QK_K // 4))# [scale, qs, d, dmin]return np.concatenate([output_scale, output_qs, output_d, output_dmin], axis=-1)-@register_qtype("q3_k")-def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):+def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape(nb, QK_K // 16, 16)@@ -498,13 +494,12 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleall_L = (torch.round(blocks * reverse_qdq_scale.unsqueeze(-1)).clip(-4, 3) + 4).to(torch.uint8)- all_L = all_L.cpu().numpy()- output_hmask = np.bitwise_or.reduce(- all_L.reshape(nb, 8, 32) >> 2 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1) # pylint: disable=E1121- all_L = np.where(all_L > 3, all_L - 4, all_L)+ output_hmask = all_L.reshape(nb, 8, 32) >> 2 << torch.arange(8, device=all_L.device).reshape(1, 8, 1)+ output_hmask = np.bitwise_or.reduce(output_hmask.cpu().numpy(), axis=1, dtype=np.uint8) # pylint: disable=E1121+ all_L = torch.where(all_L > 3, all_L - 4, all_L)- output_qs = np.bitwise_or.reduce(- all_L.reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), axis=2) # pylint: disable=E1121+ output_qs = all_L.reshape(nb, 2, 4, 32) << torch.tensor([0, 2, 4, 6], device=all_L.device).reshape(1, 1, 4, 1)+ output_qs = np.bitwise_or.reduce(output_qs.cpu().numpy(), axis=2, dtype=np.uint8) # pylint: disable=E1121output_qs = output_qs.reshape(nb, 64).astype(np.uint8)dq_scale = dq_scale.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)@@ -514,20 +509,19 @@ def q3_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale@register_qtype("q4_k")-def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):+def q4_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 32, 32))- output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)- output_qs = np.empty((nb, QK_K // 64, 32), dtype=np.uint8)+ output_scale = torch.empty((nb, K_SCALE_SIZE), dtype=torch.uint8, device=blocks.device)if scale is not None:scales = scale.reshape(-1, QK_K // 32)- mins = wmin_m.reshape(-1, QK_K // 32)+ mins = wmin.reshape(-1, QK_K // 32)output_d = d_scale.reshape(-1, 1).to(torch.float32)- output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)inv_scales = torch.where(output_d == 0, 0, 1 / output_d)- inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)+ inv_mins = torch.where(d_wmin == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)all_L = torch.zeros_like(blocks, dtype=torch.uint8)@@ -542,18 +536,18 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_winv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)- q_scales = torch.round(inv_scales * scales).clip(0, 63)- q_mins = torch.round(inv_mins * mins).clip(0, 63)+ q_scales = torch.round(inv_scales * scales).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(inv_mins * mins).clip(0, 63).to(torch.uint8)if d_scale is None:output_d = max_scales / 63- if d_wmin_m is None:+ if d_wmin is None:output_dmin = max_mins / 63d_tmp = output_d * q_scalesdm_tmp = output_dmin * q_mins- q_scales = q_scales.cpu().numpy().astype(np.uint8)- q_mins = q_mins.cpu().numpy().astype(np.uint8)+ # q_scales = q_scales.cpu().numpy().astype(np.uint8)+ # q_mins = q_mins.cpu().numpy().astype(np.uint8)output_scale[:, :4] = q_scales[:, :4]output_scale[:, 4:8] = q_mins[:, :4]@@ -565,32 +559,32 @@ def q4_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wall_L[replace_ids] = torch.round((blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) / d_tmp[replace_ids].unsqueeze(-1)).clip(0, 15).to(torch.uint8)- all_L = np.clip(all_L.cpu().numpy(), 0, 15).astype(np.uint8)output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)output_d = output_d.cpu().numpy()output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)output_dmin = output_dmin.cpu().numpy()output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)- output_qs = output_qs.reshape(nb, QK_K // 2)+ output_qs = output_qs.reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)+ output_scale = output_scale.cpu().numpy().astype(np.uint8)# [d, dmin, scale, qs]return np.concatenate([output_d, output_dmin, output_scale, output_qs], axis=-1)@register_qtype("q5_k")-def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):+def q5_k_quant_block(blocks, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 32, 32))- output_scale = np.empty((nb, K_SCALE_SIZE), dtype=np.uint8)+ output_scale = torch.empty((nb, K_SCALE_SIZE), dtype=torch.uint8, device=blocks.device)if scale is not None:scales = scale.reshape(-1, QK_K // 32)- mins = wmin_m.reshape(-1, QK_K // 32)+ mins = wmin.reshape(-1, QK_K // 32)output_d = d_scale.reshape(-1, 1).to(torch.float32)- output_dmin = d_wmin_m.reshape(-1, 1).to(torch.float32)+ output_dmin = d_wmin.reshape(-1, 1).to(torch.float32)inv_scales = torch.where(output_d == 0, 0, 1 / output_d)- inv_mins = torch.where(d_wmin_m == 0, 0, 1 / output_dmin)+ inv_mins = torch.where(d_wmin == 0, 0, 1 / output_dmin)max_scales = torch.max(scales, dim=-1, keepdim=True)[0] # (nb, 1)max_mins = torch.max(mins, dim=-1, keepdim=True)[0] # (nb, 1)all_L = torch.zeros_like(blocks, dtype=torch.uint8)@@ -600,20 +594,20 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmax_mins = torch.max(mins, dim=-1, keepdim=True)[0]inv_scales = torch.where(max_scales > 0, 63. / max_scales, 0)inv_mins = torch.where(max_mins > 0, 63. / max_mins, 0)-- q_scales = torch.round(inv_scales * scales).clip(0, 63)- q_mins = torch.round(inv_mins * mins).clip(0, 63)++ q_scales = torch.round(inv_scales * scales).clip(0, 63).to(torch.uint8)+ q_mins = torch.round(inv_mins * mins).clip(0, 63).to(torch.uint8)if d_scale is None:output_d = max_scales / 63- if d_wmin_m is None:+ if d_wmin is None:output_dmin = max_mins / 63d_tmp = output_d * q_scalesdm_tmp = output_dmin * q_mins- q_scales = q_scales.cpu().numpy().astype(np.uint8)- q_mins = q_mins.cpu().numpy().astype(np.uint8)+ # q_scales = q_scales.cpu().numpy().astype(np.uint8)+ # q_mins = q_mins.cpu().numpy().astype(np.uint8)output_scale[:, :4] = q_scales[:, :4]output_scale[:, 4:8] = q_mins[:, :4]@@ -625,23 +619,27 @@ def q5_k_quant_block(blocks, scale=None, zp=None, wmin_m=None, d_scale=None, d_wall_L[replace_ids] = torch.round((blocks[replace_ids] + dm_tmp[replace_ids].unsqueeze(-1)) \/ d_tmp[replace_ids].unsqueeze(-1)).clip(0, 31).to(torch.uint8)- all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 31) # (nb, 8, 32)+ # all_L = np.clip(all_L.cpu().numpy().astype(np.uint8), 0, 31) # (nb, 8, 32)+ all_L = torch.clamp(all_L, 0, 31)output_qs = all_L[:, ::2] | (all_L[:, 1::2] << 4)+ output_qh = all_L >> 4 << torch.arange(8, device=all_L.device).reshape(1, 8, 1)output_qh = np.bitwise_or.reduce(- all_L >> 4 << np.arange(8, dtype=np.uint8).reshape(1, 8, 1), axis=1).astype(np.uint8) # pylint: disable=E1121+ output_qh.cpu().numpy(), axis=1, dtype=np.uint8).astype(np.uint8) # pylint: disable=E1121output_d = output_d.cpu().numpy()output_d = output_d.reshape(-1, 1).astype(np.float16).view(np.uint8)output_dmin = output_dmin.cpu().numpy()output_dmin = output_dmin.reshape(-1, 1).astype(np.float16).view(np.uint8)- output_qs = output_qs.reshape(nb, QK_K // 2)+ output_qs = output_qs.reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)+ output_scale = output_scale.cpu().numpy().astype(np.uint8)+# [d, dmin, scale, qh, qs]return np.concatenate([output_d, output_dmin, output_scale, output_qh, output_qs], axis=-1)@register_qtype("q6_k")-def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scale=None, d_wmin_m=None, **kwargs):+def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin=None, d_scale=None, d_wmin=None, **kwargs):nb = blocks.shape[0]blocks = blocks.reshape((nb, QK_K // 16, 16))@@ -664,12 +662,14 @@ def q6_k_quant_block(blocks: np.array, scale=None, zp=None, wmin_m=None, d_scaleall_L[replace_ids] = torch.round(blocks[replace_ids] / d_tmp[replace_ids].reshape(-1, 1) + 32) \.clip(0, 63).to(torch.uint8)- all_L = all_L.cpu().numpy()+ # all_L = all_L.cpu().numpy()tmp_L = all_L.reshape(nb, 4, 64) & 0xF- output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2)+ output_ql = (tmp_L[:, ::2] | (tmp_L[:, 1::2] << 4)).reshape(nb, QK_K // 2).cpu().numpy().astype(np.uint8)+ ouptut_qh = (all_L >> 4).reshape(nb, 2, 4, 32) << torch.tensor(+ [0, 2, 4, 6], device=all_L.device).reshape(1, 1, 4, 1)output_qh = np.bitwise_or.reduce(- (all_L >> 4).reshape(nb, 2, 4, 32) << np.array([0, 2, 4, 6]).reshape(1, 1, 4, 1), # pylint: disable=E1121- axis=2).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121+ ouptut_qh.cpu().numpy(), # pylint: disable=E1121+ axis=2, dtype=np.uint8).reshape(nb, QK_K // 4).astype(np.uint8) # pylint: disable=E1121output_d = output_d.cpu().numpy().reshape(-1, 1).astype(np.float16).view(np.uint8)diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 4a2781a82..a923a37ec 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -1584,6 +1584,20 @@ def _get_digital_in_layer_name(layer_name):else:return None+def _search_gguf_type(gguf_type):+ if gguf_type in GGUF_INNER_CONFIG:+ return gguf_type+ pattern = re.compile("gguf:q([0-9]{1,})_[01k]")+ bits = re.search(pattern, gguf_type)+ if not bits:+ raise KeyError(f"{gguf_type} is not a correct gguf type, please check")++ for suffix in ["_0", "_1", "_k"]:+ if gguf_type.endswith(suffix):+ continue+ if (tmp_type := re.sub("_[01k]", suffix, gguf_type)) in GGUF_INNER_CONFIG:+ return tmp_type+ return None##https://github.com/ggml-org/llama.cpp/blob/9e31bec4fd53634c9e5b04650488a09a055f5dab/src/llama-quant.cpp#L129def get_layer_config_by_gguf_format(layer_config, gguf_format, model):@@ -1651,17 +1665,30 @@ def get_layer_config_by_gguf_format(layer_config, gguf_format, model):input_features = layer.weight.shape[-1]i_layer = _get_digital_in_layer_name(layer_name)+ if lm_head_name is not None and layer_name == lm_head_name:+ target_bits = int(re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]['lm_head']).group(1))+ if isinstance(layer, torch.nn.Embedding):+ target_bits = int(+ re.search("gguf:q([0-9]{1,})_[01k]", GGUF_CONFIG[target_gguf_format]['embedding']).group(1))+gguf_name = tensor_map.get_name(layer_name)- if target_bits is not None and "bits" in config and config["bits"] != target_bits:- bits_index = 6+ bits_index = 6+ if config.get("fixed_by_user", False):+ if "bits" not in config:+ logger.warning(+ f"Setting layer_config requires providing bits, {layer_name} has not bits,"+ f" using bits={target_bits} instead.")+ new_type = new_type[:bits_index] + target_bits + new_type[bits_index + 1:]+ else:+ new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1:]+ new_type = _search_gguf_type(new_type)+ if new_type is None:+ raise ValueError(f"invalid bit setting for {layer_name}")+ elif (target_bits is not None and+ "bits" in config and config["bits"] != target_bits):new_type = new_type[:bits_index] + str(config["bits"]) + new_type[bits_index + 1:]- if not new_type in GGUF_INNER_CONFIG:- for tmp_type in (new_type[:bits_index + 1] + "_k", new_type[:bits_index + 1] + "_1",- new_type[:bits_index + 1] + "_0"):- if tmp_type in GGUF_INNER_CONFIG:- new_type = tmp_type- break- if not new_type in GGUF_INNER_CONFIG:+ new_type = _search_gguf_type(new_type)+ if new_type is None:raise ValueError(f"invalid bit setting for {layer_name}")elif lm_head_name is not None and layer_name == lm_head_name and not tie_word_embeddings:if gguf.MODEL_ARCH.FALCON == model_class.model_arch or input_features % block_size != 0:@@ -1845,4 +1872,3 @@ def get_gguf_qtype_by_layer_config(layer_config):if bits == 8 and sym and group_size == 32:return gguf.GGMLQuantizationType.Q8_0raise ValueError(f"Unknown layer config")-diff --git a/test/test_cpu/test_gguf_format.py b/test/test_cpu/test_gguf_format.pyindex b58d3acc2..6a5192fdd 100644--- a/test/test_cpu/test_gguf_format.py+++ b/test/test_cpu/test_gguf_format.py@@ -237,6 +237,39 @@ class TestGGUF(unittest.TestCase):shutil.rmtree("./saved", ignore_errors=True)+ def test_q4_k_m(self):+ model_name = "Qwen/Qwen2.5-7B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ layer_config = {+ "lm_head": {'bits': 4, 'group_size': 32, 'sym': False, 'data_type': 'int_asym_dq', 'scale_dtype': torch.float32, 'super_bits': 6, 'super_group_size': 8, 'in_blocks': False},+ "model.embed_tokens": {'bits': 6, 'group_size': 32, 'super_bits': 6, 'super_group_size': 8},+ "model.layers.12.mlp.gate_proj": {'bits': 3},+ "model.layers.10.mlp.gate_proj": {'bits': 8},+ }+ autoround = AutoRound(+ model,+ tokenizer,+ layer_config=layer_config,+ iters=0,+ seqlen=1,+ dataset=self.llm_dataloader,+ disable_opt_rtn=True+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")+ self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16)+ self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "int_sym_dq")+ self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "int_asym_dq")+ self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6)+ self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4)+ self.assertEqual(autoround.model.model.embed_tokens.bits, 6)+ self.assertEqual(autoround.model.model.embed_tokens.group_size, 16)+ self.assertEqual(autoround.model.model.layers[12].mlp.gate_proj.bits, 3)+ self.assertEqual(autoround.model.model.layers[10].mlp.gate_proj.bits, 8)+ self.assertEqual(autoround.layer_config['model.layers.10.mlp.gate_proj']['mostly'], "gguf:q8_0")+ shutil.rmtree("./saved", ignore_errors=True)+if __name__ == "__main__":unittest.main()diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.pyindex 022508213..405596c89 100644--- a/test/test_cuda/test_gguf.py+++ b/test/test_cuda/test_gguf.py@@ -171,33 +171,5 @@ class TestAutoRound(unittest.TestCase):self.assertGreater(result['results']['piqa']['acc,none'], 0.55)shutil.rmtree("./saved", ignore_errors=True)- @require_gguf- def test_q4_k_m(self):- model_name = "/models/Qwen2.5-1.5B-Instruct"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- bits, group_size, sym, super_group_size, super_bits = 4, 32, False, 8, 6- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=0,- seqlen=1,- dataset=self.llm_dataloader,- data_type="int_asym_dq",- super_group_size=super_group_size,- super_bits=super_bits- )- quantized_model_path = "./saved"- autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")- self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16)- self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "gguf_int_sym_dq")- self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "gguf_int_asym_dq")- self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6)- self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4)- shutil.rmtree("./saved", ignore_errors=True)-if __name__ == "__main__":unittest.main()\ No newline at end of file