PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
support for q4_k_m (#596)
Heng Guof3fca3033151
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 76a5f9320..ebc39a8ac 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -53,7 +53,8 @@ from .utils import (reset_params,init_cache, check_skippable_keywords, get_shared_keys, SUPPORTED_DTYPES, infer_bits_by_data_type,_gguf_args_check,- check_seqlen_compatible+ check_seqlen_compatible,+ get_layer_config_by_gguf_format)from .low_cpu_mem.utils import get_layers_before_block@@ -485,6 +486,10 @@ class AutoRound(object):"""# Validate and process the specified formats_gguf_args_check(self, format)+ if "gguf" in format:+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(self.layer_config, format,+ self.model)+ self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')from auto_round.utils import SUPPORTED_FORMATSfor format_ in formats:diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.pyindex 8cc3f85e2..f82853f32 100644--- a/auto_round/export/export_to_gguf/config.py+++ b/auto_round/export/export_to_gguf/config.py@@ -14,29 +14,27 @@GGUF_CONFIG = {}-GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}+GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "sym": True, "data_type": "int"}GGUF_CONFIG["gguf:q4_1"] = {"bits": 4,"act_bits": 16,"group_size": 32,- "asym": True,"sym": False,"data_type": "int_asym_float_zp"}-GGUF_CONFIG["gguf:q5_0"] = {"bits": 5, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}+GGUF_CONFIG["gguf:q5_0"] = {"bits": 5, "act_bits": 16, "group_size": 32, "sym": True, "data_type": "int"}GGUF_CONFIG["gguf:q5_1"] = {"bits": 5,"act_bits": 16,"group_size": 32,- "asym": True,"sym": False,"data_type": "int_asym_float_zp"}-GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}+GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "sym": True, "data_type": "int"}GGUF_CONFIG["gguf:q2_k_s"] = {@@ -45,7 +43,6 @@ GGUF_CONFIG["gguf:q2_k_s"] = {"super_group_size": 16,"super_bits": 4,"group_size": 16,- "asym": True,"sym": False,"data_type": "int_asym_dq"}@@ -56,18 +53,16 @@ GGUF_CONFIG["gguf:q3_k_s"] = {"super_group_size": 16,"super_bits": 6,"group_size": 16,- "asym": False,"sym": True,"data_type": "int_sym_dq"}-GGUF_CONFIG["gguf:q4_k_s"] = {+GGUF_CONFIG["gguf:q4_k_m"] = GGUF_CONFIG["gguf:q4_k_s"] = {"bits": 4,"act_bits": 16,"super_group_size": 8,"super_bits": 6,"group_size": 32,- "asym": True,"sym": False,"data_type": "int_asym_dq"}@@ -78,7 +73,6 @@ GGUF_CONFIG["gguf:q5_k_s"] = {"super_group_size": 8,"super_bits": 6,"group_size": 32,- "asym": True,"sym": False,"data_type": "int_asym_dq"}@@ -89,7 +83,6 @@ GGUF_CONFIG["gguf:q6_k"] = GGUF_CONFIG["gguf:q6_k_s"] = {"super_group_size": 16,"super_bits": 8,"group_size": 16,- "asym": False,"sym": True,"data_type": "int_sym_dq"}diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.pyindex 6cdd5efd3..085a91a36 100644--- a/auto_round/export/export_to_gguf/convert.py+++ b/auto_round/export/export_to_gguf/convert.py@@ -67,7 +67,6 @@ if 'NO_LOCAL_GGUF' not in os.environ:###### MODEL DEFINITIONS ######-class SentencePieceTokenTypes(IntEnum):NORMAL = 1UNKNOWN = 2@@ -1092,10 +1091,10 @@ class OriModel:class Model(OriModel):-def __init__(self,model,+ layer_config: dict,dir_model: Path,ftype: gguf.LlamaFileType,fname_out: Path,@@ -1110,6 +1109,7 @@ class Model(OriModel):small_first_shard: bool = False,hparams: dict[str, Any] | None = None):self.model = model+ self.layer_config = layer_configsuper().__init__(dir_model=dir_model,ftype=ftype,@@ -1180,6 +1180,37 @@ class Model(OriModel):data = data_torch.squeeze().cpu().numpy()return data, data_qtype+ def get_qtype_by_layer_config(self, layer_config, name, data_qtype):+ name = name[:-len('.weight')]+ if name not in layer_config or layer_config[name]['bits'] >= 16:+ return data_qtype+ bits = layer_config[name]['bits']+ data_qtype = re.sub(r"Q\d", f"Q{bits}", self.ftype.name)++ if data_qtype == "MOSTLY_Q8_0":+ data_qtype = gguf.GGMLQuantizationType.Q8_0+ elif data_qtype == "MOSTLY_Q4_0":+ data_qtype = gguf.GGMLQuantizationType.Q4_0+ elif data_qtype == "MOSTLY_Q4_1":+ data_qtype = gguf.GGMLQuantizationType.Q4_1+ elif data_qtype == "MOSTLY_Q5_0":+ data_qtype = gguf.GGMLQuantizationType.Q5_0+ elif data_qtype == "MOSTLY_Q5_1":+ data_qtype = gguf.GGMLQuantizationType.Q5_1+ elif data_qtype.startswith("MOSTLY_Q2_K"):+ data_qtype = gguf.GGMLQuantizationType.Q2_K+ elif data_qtype.startswith("MOSTLY_Q3_K"):+ data_qtype = gguf.GGMLQuantizationType.Q3_K+ elif data_qtype.startswith("MOSTLY_Q4_K"):+ data_qtype = gguf.GGMLQuantizationType.Q4_K+ elif data_qtype.startswith("MOSTLY_Q5_K"):+ data_qtype = gguf.GGMLQuantizationType.Q5_K+ elif data_qtype.startswith("MOSTLY_Q6_K"):+ data_qtype = gguf.GGMLQuantizationType.Q6_K+ else:+ raise ValueError(f"Unknown file type: {data_qtype}")+ return data_qtype+def prepare_tensors(self):max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")@@ -1241,6 +1272,9 @@ class Model(OriModel):# TODO: use Q4_K and Q6_Kdata_qtype = gguf.GGMLQuantizationType.F16# data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type++ # get data_qtype by layer_config+ data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)# No override (data_qtype is False), or wants to be quantized (data_qtype is True)if isinstance(data_qtype, bool):@@ -1266,6 +1300,8 @@ class Model(OriModel):data_qtype = gguf.GGMLQuantizationType.Q3_Kelif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:data_qtype = gguf.GGMLQuantizationType.Q4_K+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:+ data_qtype = gguf.GGMLQuantizationType.Q4_Kelif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:data_qtype = gguf.GGMLQuantizationType.Q5_Kelif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:@@ -5981,4 +6017,4 @@ def main() -> None:if __name__ == '__main__':- main()+ main()\ No newline at end of filediff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.pyindex d7cfc00db..527085e5e 100644--- a/auto_round/export/export_to_gguf/export.py+++ b/auto_round/export/export_to_gguf/export.py@@ -34,14 +34,18 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {"q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,"q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,"q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,+ "q3_k_m": gguf.LlamaFileType.MOSTLY_Q3_K_M,+ "q3_k_l": gguf.LlamaFileType.MOSTLY_Q3_K_L,"q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,+ "q4_k_m": gguf.LlamaFileType.MOSTLY_Q4_K_M,"q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,+ "q5_k_m": gguf.LlamaFileType.MOSTLY_Q5_K_M,"q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,"q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,"auto": gguf.LlamaFileType.GUESSED,}-def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, **kwargs):"""Export the model to gguf format."""if output_dir is not None and os.path.exists(output_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")@@ -80,6 +84,7 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):model_instance = model_class(model,+ layer_config,dir_model=tmp_work_dir,ftype=output_type,fname_out=Path(output_dir),diff --git a/auto_round/utils.py b/auto_round/utils.pyindex e9ae56cac..0516c70f0 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -46,7 +46,7 @@ class SupportedFormats:def __str__(self):return "(%s)" % ', '.join(self._support_format + ("gguf:q*_0", "gguf:q*_1", "gguf:q*_k_s"))-+def __getitem__(self, key):return self._support_list[key]@@ -1194,7 +1194,7 @@ def _gguf_args_check(args_or_ar, format_str=None):sys.exit(-1)if re.search(pattern, format):if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:- logger.error(f"Cannot eport {pre_dq_format} and {format} at the same time.")+ logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")sys.exit(-1)else:pre_dq_format = format@@ -1229,6 +1229,9 @@ def _gguf_args_check(args_or_ar, format_str=None):v = "int"if re.search("q\d_k", format) and args_or_ar.iters == 0:v = f"gguf_{v}"+ if k == "sym" and isinstance(args_or_ar, argparse.Namespace):+ k = "asym"+ v = not vif getattr(args_or_ar, k) != v:unsupport_list.append(f"{k}={getattr(args_or_ar, k)}")reset_list.append(f"{k}={v}")@@ -1554,3 +1557,96 @@ def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):raise ValueError(f"seqlen({input_seqlen}) exceeds tokenizer.model_max_length({tokenizer.model_max_length}). " \"Please oncider Consider lowering the '--seqlen' or increasing tokenizer.model_max_length.")+def _use_more_bits(i_layer: int, n_layer: int):+ return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer//8) % 3 == 2)++def _get_digital_in_layer_name(layer_name):+ pattern = re.compile("([a-zA-Z]+\.){1,}(\d+)")+ res = re.search(pattern, layer_name)+ if res:+ return int(res[2])+ else:+ return None++def get_layer_config_by_gguf_format(layer_config, gguf_format, model):+ # TODO: support for other format later+ if "q4_k_m" not in gguf_format:+ return layer_config, {}++ import gguf # pylint: disable=E0401+ from auto_round.export.export_to_gguf.convert import Model+ from auto_round.export.export_to_gguf.config import GGUF_CONFIG+ model_architecture = model.config.architectures[0]+ try:+ model_class = Model.from_model_architecture(model_architecture)+ except NotImplementedError:+ return layer_config, {}++ n_layer = None+ for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"]:+ if hasattr(model.config, name):+ n_layer = getattr(model.config, name)+ if n_layer is None:+ return layer_config, {}++ tensor_map = gguf.get_tensor_name_map(model_class.model_arch, n_layer)++ def _set_config(config, gguf_config):+ for k, v in gguf_config.items():+ if k in config:+ if k == "data_type" and config[k].startswith("gguf_"):+ config[k] = "gguf_" + v+ else:+ config[k] = v+ return config++ gguf_format_config = {}+ for layer_name, config in layer_config.items():+ if config["bits"] >= 16:+ continue+ gguf_format_config[layer_name] = gguf_format+ i_layer = _get_digital_in_layer_name(layer_name)+ if i_layer is None:+ continue+ gguf_name = tensor_map.get_name(layer_name)+ if gguf_name is None:+ continue+ # attn_v+ if "attn_v" in gguf_name:+ if _use_more_bits(i_layer, n_layer):+ # q6_k+ config = _set_config(config, GGUF_CONFIG["gguf:q6_k"])+ gguf_format_config[layer_name] = "gguf:q6_k"++ # ffn_down+ if "ffn_down" in gguf_name:+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch:+ if i_layer < n_layer // 16:+ format = "gguf:q6_k"+ elif _use_more_bits(i_layer, n_layer):+ format = "gguf:q5_k_s"+ else:+ format = "gguf:q4_k_s"+ config = _set_config(config, GGUF_CONFIG[format])+ gguf_format_config[layer_name] = format+ else:+ if _use_more_bits(i_layer, n_layer):+ config = _set_config(config, GGUF_CONFIG["gguf:q6_k"])+ gguf_format_config[layer_name] = "gguf:q6_k"++ # attn_output+ if "attn_output" in gguf_name and gguf.MODEL_ARCH.FALCON != model_class.model_arch:+ n_export = 0+ for name in ["num_experts", "num_local_experts", "n_routed_experts"]:+ if hasattr(model.config, name):+ n_export = getattr(model.config, name)+ if n_export == 8:+ config = _set_config(config, GGUF_CONFIG["gguf:q5_k_s"])+ gguf_format_config[layer_name] = "gguf:q5_k_s"++ # attn_qkv+ if "attn_qkv" in gguf_name:+ config = _set_config(config, GGUF_CONFIG["gguf:q5_k_s"])++ layer_config[layer_name] = config+ return layer_config, gguf_format_config\ No newline at end of filediff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.pyindex 430446726..022508213 100644--- a/test/test_cuda/test_gguf.py+++ b/test/test_cuda/test_gguf.py@@ -170,6 +170,34 @@ class TestAutoRound(unittest.TestCase):result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="piqa")self.assertGreater(result['results']['piqa']['acc,none'], 0.55)shutil.rmtree("./saved", ignore_errors=True)++ @require_gguf+ def test_q4_k_m(self):+ model_name = "/models/Qwen2.5-1.5B-Instruct"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ bits, group_size, sym, super_group_size, super_bits = 4, 32, False, 8, 6+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=0,+ seqlen=1,+ dataset=self.llm_dataloader,+ data_type="int_asym_dq",+ super_group_size=super_group_size,+ super_bits=super_bits+ )+ quantized_model_path = "./saved"+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")+ self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16)+ self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "gguf_int_sym_dq")+ self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "gguf_int_asym_dq")+ self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6)+ self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4)+ shutil.rmtree("./saved", ignore_errors=True)if __name__ == "__main__":unittest.main()\ No newline at end of file