PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
gguf add support for Mistral-Magistral-Devstral and Granite (#1226)
Heng Guo6f2f9b9ecbb0
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 96847c2ea..1b7e0735d 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -80,7 +80,7 @@ def download_convert_file(redownload=False):f" from https://github.com/ggml-org/llama.cpp manually and move it to {gguf_export_dir}.")sys.exit(-1)- with open(os.path.join(gguf_export_dir, FILE_NAME), "w") as f:+ with open(os.path.join(gguf_export_dir, FILE_NAME), "w", encoding="utf-8") as f:f.write(response.text)@@ -214,12 +214,32 @@ def _quant_data_with_args(return data-def _quant_data(cls, data_torch, data_qtype, name, modify_name, bid, device=None):+def _quant_data(cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=None):+ """++ Args:+ data_torch: original data tensor+ data_qtype: quantization type+ name: original tensor name, for getting auto_round config, model.language_model.layers.0.input_linear.weight+ modify_name: modified tensor name, for gguf mapping, model.layers.0.input_linear.weight+ new_name: name after modify_tensors, gguf using this to save tensor, like blk.0.ffn_gate.weight+ bid: block id+ device: device to perform quantization. Defaults to None.++ """suffix = ".weight"device = data_torch.device if device is None else deviceif suffix in name:layer_name = name[: -len(suffix)]module = get_module(cls.model, layer_name)+ kwargs = {+ "scale": None,+ "zp": None,+ "d_scale": None,+ "d_wmin": None,+ "wmin": None,+ "imatrix": None,+ }if hasattr(module, "scale"):if hasattr(cls, "permute"):bs = module.weight.shape[0]@@ -228,34 +248,15 @@ def _quant_data(cls, data_torch, data_qtype, name, modify_name, bid, device=Noneattr_tensor = getattr(module, attr)if not isinstance(attr_tensor, torch.Tensor):continue- ori_shape = attr_tensor.shape- attr_tensor = cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid)[0][1]- attr_tensor = attr_tensor.reshape(ori_shape)- setattr(module, attr, attr_tensor)- scale = module.scale if hasattr(module, "scale") else None- zp = module.zp if hasattr(module, "zp") else None+ attr_tensors_dict = dict(cls.modify_tensors(attr_tensor.reshape(bs, -1), modify_name, bid))+ attr_tensor = attr_tensors_dict[new_name]+ if attr in kwargs:+ kwargs[attr] = attr_tensor.to(torch.float32)+ else:+ kwargs[attr.replace("w_", "")] = attr_tensor.to(torch.float32)data_torch = data_torch.to(torch.float32)- scale = scale.to(torch.float32) if isinstance(scale, torch.Tensor) else scale- zp = zp.to(torch.float32) if isinstance(zp, torch.Tensor) else zp- if data_qtype.name.lower().endswith("_k"):- d_scale = module.w_d_scale.to(torch.float32) if hasattr(module, "w_d_scale") else None- d_wmin = module.w_d_wmin.to(torch.float32) if hasattr(module, "w_d_wmin") else None- wmin = module.w_wmin.to(torch.float32) if hasattr(module, "w_wmin") else None- imatrix = module.imatrix.to(torch.float32) if hasattr(module, "imatrix") else None-- data = ggml_quant(- data_torch,- data_qtype.name.lower(),- scale,- zp,- wmin=wmin,- d_scale=d_scale,- d_wmin=d_wmin,- imatrix=imatrix,- device=device,- )- else:- data = ggml_quant(data_torch, data_qtype.name.lower(), scale, zp, device=device)++ data = ggml_quant(data_torch, data_qtype.name.lower(), device=device, **kwargs)else:# if data_torch.dtype ==torch.float32:# data_qtype = gguf.GGMLQuantizationType.F32@@ -583,12 +584,16 @@ def prepare_tensors(cls):if arr_name[i].isdecimal() and int(arr_name[i]) == (data_torch.shape[0] - 1):arr_name[i] = str(idx)arr_name = ".".join(arr_name)- arr, data_qtype = _quant_data(cls, arr, data_qtype, arr_name, modify_name, bid, device=device)+ arr, data_qtype = _quant_data(+ cls, arr, data_qtype, arr_name, modify_name, new_name, bid, device=device+ )new_data.append(arr)data = np.array(new_data)del new_dataelse:- data, data_qtype = _quant_data(cls, data_torch, data_qtype, name, modify_name, bid, device=device)+ data, data_qtype = _quant_data(+ cls, data_torch, data_qtype, name, modify_name, new_name, bid, device=device+ )shape = gguf.quant_shape_from_byte_shape(data.shape, data_qtype) if data.dtype == np.uint8 else data.shapediff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex df7264dd4..c1add0ed5 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -99,7 +99,11 @@ def create_model_class(raise TypeError(f"{output_type} type is not supported")output_type = FTYPE_MAP.get(output_type.lower())- hparams = convert_hf_to_gguf.ModelBase.load_hparams(Path(tmp_work_dir), "mistral" in model.config.model_type)+ if "mistral" in model.config.model_type and "params.json" in os.listdir(tmp_work_dir):+ is_mistral_format = True+ else:+ is_mistral_format = False+ hparams = convert_hf_to_gguf.ModelBase.load_hparams(Path(tmp_work_dir), is_mistral_format)hparams.pop("quantization_config", None)model_instance = model_class(dir_model=Path(tmp_work_dir),diff --git a/auto_round/utils/model.py b/auto_round/utils/model.pyindex 4a40e9626..8b85bc26e 100644--- a/auto_round/utils/model.py+++ b/auto_round/utils/model.py@@ -810,7 +810,10 @@ def get_gguf_architecture(dir_model, model_type=ModelType.TEXT):tmp_model_type = hparams.model_typeif "mistral" == tmp_model_type:is_mistral_format = True- hparams = ModelBase.load_hparams(dir_model, is_mistral_format)+ try:+ hparams = ModelBase.load_hparams(dir_model, is_mistral_format)+ except Exception:+ is_mistral_format = Falseif not is_mistral_format:model_class = get_model_architecture(hparams, model_type)elif model_type == ModelType.MMPROJ: