Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

support for q4_k_m (#596)

Heng Guof3fca3033151
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 76a5f9320..ebc39a8ac 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -53,7 +53,8 @@ from .utils import (
reset_params,
init_cache, check_skippable_keywords, get_shared_keys, SUPPORTED_DTYPES, infer_bits_by_data_type,
_gguf_args_check,
- check_seqlen_compatible
+ check_seqlen_compatible,
+ get_layer_config_by_gguf_format
)
from .low_cpu_mem.utils import get_layers_before_block
@@ -485,6 +486,10 @@ class AutoRound(object):
"""
# Validate and process the specified formats
_gguf_args_check(self, format)
+ if "gguf" in format:
+ self.layer_config, gguf_format_config = get_layer_config_by_gguf_format(self.layer_config, format,
+ self.model)
+ self.has_qlayer_outside_block = self.set_layerwise_config(self.layer_config)
formats = format.replace("q*_", f"q{self.bits}_").replace(' ', '').split(',')
from auto_round.utils import SUPPORTED_FORMATS
for format_ in formats:
diff --git a/auto_round/export/export_to_gguf/config.py b/auto_round/export/export_to_gguf/config.py
index 8cc3f85e2..f82853f32 100644
--- a/auto_round/export/export_to_gguf/config.py
+++ b/auto_round/export/export_to_gguf/config.py
@@ -14,29 +14,27 @@
GGUF_CONFIG = {}
-GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}
+GGUF_CONFIG["gguf:q4_0"] = {"bits": 4, "act_bits": 16, "group_size": 32, "sym": True, "data_type": "int"}
GGUF_CONFIG["gguf:q4_1"] = {
"bits": 4,
"act_bits": 16,
"group_size": 32,
- "asym": True,
"sym": False,
"data_type": "int_asym_float_zp"
}
-GGUF_CONFIG["gguf:q5_0"] = {"bits": 5, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}
+GGUF_CONFIG["gguf:q5_0"] = {"bits": 5, "act_bits": 16, "group_size": 32, "sym": True, "data_type": "int"}
GGUF_CONFIG["gguf:q5_1"] = {
"bits": 5,
"act_bits": 16,
"group_size": 32,
- "asym": True,
"sym": False,
"data_type": "int_asym_float_zp"
}
-GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "asym": False, "sym": True, "data_type": "int"}
+GGUF_CONFIG["gguf:q8_0"] = {"bits": 8, "act_bits": 16, "group_size": 32, "sym": True, "data_type": "int"}
GGUF_CONFIG["gguf:q2_k_s"] = {
@@ -45,7 +43,6 @@ GGUF_CONFIG["gguf:q2_k_s"] = {
"super_group_size": 16,
"super_bits": 4,
"group_size": 16,
- "asym": True,
"sym": False,
"data_type": "int_asym_dq"
}
@@ -56,18 +53,16 @@ GGUF_CONFIG["gguf:q3_k_s"] = {
"super_group_size": 16,
"super_bits": 6,
"group_size": 16,
- "asym": False,
"sym": True,
"data_type": "int_sym_dq"
}
-GGUF_CONFIG["gguf:q4_k_s"] = {
+GGUF_CONFIG["gguf:q4_k_m"] = GGUF_CONFIG["gguf:q4_k_s"] = {
"bits": 4,
"act_bits": 16,
"super_group_size": 8,
"super_bits": 6,
"group_size": 32,
- "asym": True,
"sym": False,
"data_type": "int_asym_dq"
}
@@ -78,7 +73,6 @@ GGUF_CONFIG["gguf:q5_k_s"] = {
"super_group_size": 8,
"super_bits": 6,
"group_size": 32,
- "asym": True,
"sym": False,
"data_type": "int_asym_dq"
}
@@ -89,7 +83,6 @@ GGUF_CONFIG["gguf:q6_k"] = GGUF_CONFIG["gguf:q6_k_s"] = {
"super_group_size": 16,
"super_bits": 8,
"group_size": 16,
- "asym": False,
"sym": True,
"data_type": "int_sym_dq"
}
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 6cdd5efd3..085a91a36 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -67,7 +67,6 @@ if 'NO_LOCAL_GGUF' not in os.environ:
###### MODEL DEFINITIONS ######
-
class SentencePieceTokenTypes(IntEnum):
NORMAL = 1
UNKNOWN = 2
@@ -1092,10 +1091,10 @@ class OriModel:
class Model(OriModel):
-
def __init__(
self,
model,
+ layer_config: dict,
dir_model: Path,
ftype: gguf.LlamaFileType,
fname_out: Path,
@@ -1110,6 +1109,7 @@ class Model(OriModel):
small_first_shard: bool = False,
hparams: dict[str, Any] | None = None):
self.model = model
+ self.layer_config = layer_config
super().__init__(
dir_model=dir_model,
ftype=ftype,
@@ -1180,6 +1180,37 @@ class Model(OriModel):
data = data_torch.squeeze().cpu().numpy()
return data, data_qtype
+ def get_qtype_by_layer_config(self, layer_config, name, data_qtype):
+ name = name[:-len('.weight')]
+ if name not in layer_config or layer_config[name]['bits'] >= 16:
+ return data_qtype
+ bits = layer_config[name]['bits']
+ data_qtype = re.sub(r"Q\d", f"Q{bits}", self.ftype.name)
+
+ if data_qtype == "MOSTLY_Q8_0":
+ data_qtype = gguf.GGMLQuantizationType.Q8_0
+ elif data_qtype == "MOSTLY_Q4_0":
+ data_qtype = gguf.GGMLQuantizationType.Q4_0
+ elif data_qtype == "MOSTLY_Q4_1":
+ data_qtype = gguf.GGMLQuantizationType.Q4_1
+ elif data_qtype == "MOSTLY_Q5_0":
+ data_qtype = gguf.GGMLQuantizationType.Q5_0
+ elif data_qtype == "MOSTLY_Q5_1":
+ data_qtype = gguf.GGMLQuantizationType.Q5_1
+ elif data_qtype.startswith("MOSTLY_Q2_K"):
+ data_qtype = gguf.GGMLQuantizationType.Q2_K
+ elif data_qtype.startswith("MOSTLY_Q3_K"):
+ data_qtype = gguf.GGMLQuantizationType.Q3_K
+ elif data_qtype.startswith("MOSTLY_Q4_K"):
+ data_qtype = gguf.GGMLQuantizationType.Q4_K
+ elif data_qtype.startswith("MOSTLY_Q5_K"):
+ data_qtype = gguf.GGMLQuantizationType.Q5_K
+ elif data_qtype.startswith("MOSTLY_Q6_K"):
+ data_qtype = gguf.GGMLQuantizationType.Q6_K
+ else:
+ raise ValueError(f"Unknown file type: {data_qtype}")
+ return data_qtype
+
def prepare_tensors(self):
max_name_len = max(len(s) for _, s in self.tensor_map.mapping.values()) + len(".weight,")
@@ -1241,6 +1272,9 @@ class Model(OriModel):
# TODO: use Q4_K and Q6_K
data_qtype = gguf.GGMLQuantizationType.F16
# data_qtype = gguf.GGMLQuantizationType.Q8_0 # llama.cpp:llama_tensor_get_type
+
+ # get data_qtype by layer_config
+ data_qtype = self.get_qtype_by_layer_config(self.layer_config, name, data_qtype)
# No override (data_qtype is False), or wants to be quantized (data_qtype is True)
if isinstance(data_qtype, bool):
@@ -1266,6 +1300,8 @@ class Model(OriModel):
data_qtype = gguf.GGMLQuantizationType.Q3_K
elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_S:
data_qtype = gguf.GGMLQuantizationType.Q4_K
+ elif self.ftype == gguf.LlamaFileType.MOSTLY_Q4_K_M:
+ data_qtype = gguf.GGMLQuantizationType.Q4_K
elif self.ftype == gguf.LlamaFileType.MOSTLY_Q5_K_S:
data_qtype = gguf.GGMLQuantizationType.Q5_K
elif self.ftype == gguf.LlamaFileType.MOSTLY_Q6_K:
@@ -5981,4 +6017,4 @@ def main() -> None:
if __name__ == '__main__':
- main()
+ main()
\ No newline at end of file
diff --git a/auto_round/export/export_to_gguf/export.py b/auto_round/export/export_to_gguf/export.py
index d7cfc00db..527085e5e 100644
--- a/auto_round/export/export_to_gguf/export.py
+++ b/auto_round/export/export_to_gguf/export.py
@@ -34,14 +34,18 @@ FTYPE_MAP: dict[str, gguf.LlamaFileType] = {
"q8_0": gguf.LlamaFileType.MOSTLY_Q8_0,
"q2_k_s": gguf.LlamaFileType.MOSTLY_Q2_K_S,
"q3_k_s": gguf.LlamaFileType.MOSTLY_Q3_K_S,
+ "q3_k_m": gguf.LlamaFileType.MOSTLY_Q3_K_M,
+ "q3_k_l": gguf.LlamaFileType.MOSTLY_Q3_K_L,
"q4_k_s": gguf.LlamaFileType.MOSTLY_Q4_K_S,
+ "q4_k_m": gguf.LlamaFileType.MOSTLY_Q4_K_M,
"q5_k_s": gguf.LlamaFileType.MOSTLY_Q5_K_S,
+ "q5_k_m": gguf.LlamaFileType.MOSTLY_Q5_K_M,
"q6_k": gguf.LlamaFileType.MOSTLY_Q6_K,
"q6_k_s": gguf.LlamaFileType.MOSTLY_Q6_K,
"auto": gguf.LlamaFileType.GUESSED,
}
-def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):
+def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", layer_config=None, **kwargs):
"""Export the model to gguf format."""
if output_dir is not None and os.path.exists(output_dir):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
@@ -80,6 +84,7 @@ def save_quantized_as_gguf(output_dir, backend="gguf:q4_0", **kwargs):
model_instance = model_class(
model,
+ layer_config,
dir_model=tmp_work_dir,
ftype=output_type,
fname_out=Path(output_dir),
diff --git a/auto_round/utils.py b/auto_round/utils.py
index e9ae56cac..0516c70f0 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -46,7 +46,7 @@ class SupportedFormats:
def __str__(self):
return "(%s)" % ', '.join(self._support_format + ("gguf:q*_0", "gguf:q*_1", "gguf:q*_k_s"))
-
+
def __getitem__(self, key):
return self._support_list[key]
@@ -1194,7 +1194,7 @@ def _gguf_args_check(args_or_ar, format_str=None):
sys.exit(-1)
if re.search(pattern, format):
if pre_dq_format and re.search(pattern, format).group() not in pre_dq_format:
- logger.error(f"Cannot eport {pre_dq_format} and {format} at the same time.")
+ logger.error(f"Cannot export {pre_dq_format} and {format} at the same time.")
sys.exit(-1)
else:
pre_dq_format = format
@@ -1229,6 +1229,9 @@ def _gguf_args_check(args_or_ar, format_str=None):
v = "int"
if re.search("q\d_k", format) and args_or_ar.iters == 0:
v = f"gguf_{v}"
+ if k == "sym" and isinstance(args_or_ar, argparse.Namespace):
+ k = "asym"
+ v = not v
if getattr(args_or_ar, k) != v:
unsupport_list.append(f"{k}={getattr(args_or_ar, k)}")
reset_list.append(f"{k}={v}")
@@ -1554,3 +1557,96 @@ def check_seqlen_compatible(input_seqlen, tokenizer=None, model=None):
raise ValueError(f"seqlen({input_seqlen}) exceeds tokenizer.model_max_length({tokenizer.model_max_length}). " \
"Please oncider Consider lowering the '--seqlen' or increasing tokenizer.model_max_length.")
+def _use_more_bits(i_layer: int, n_layer: int):
+ return (i_layer < n_layer // 8) or (i_layer >= 7 * n_layer // 8) or ((i_layer - n_layer//8) % 3 == 2)
+
+def _get_digital_in_layer_name(layer_name):
+ pattern = re.compile("([a-zA-Z]+\.){1,}(\d+)")
+ res = re.search(pattern, layer_name)
+ if res:
+ return int(res[2])
+ else:
+ return None
+
+def get_layer_config_by_gguf_format(layer_config, gguf_format, model):
+ # TODO: support for other format later
+ if "q4_k_m" not in gguf_format:
+ return layer_config, {}
+
+ import gguf # pylint: disable=E0401
+ from auto_round.export.export_to_gguf.convert import Model
+ from auto_round.export.export_to_gguf.config import GGUF_CONFIG
+ model_architecture = model.config.architectures[0]
+ try:
+ model_class = Model.from_model_architecture(model_architecture)
+ except NotImplementedError:
+ return layer_config, {}
+
+ n_layer = None
+ for name in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"]:
+ if hasattr(model.config, name):
+ n_layer = getattr(model.config, name)
+ if n_layer is None:
+ return layer_config, {}
+
+ tensor_map = gguf.get_tensor_name_map(model_class.model_arch, n_layer)
+
+ def _set_config(config, gguf_config):
+ for k, v in gguf_config.items():
+ if k in config:
+ if k == "data_type" and config[k].startswith("gguf_"):
+ config[k] = "gguf_" + v
+ else:
+ config[k] = v
+ return config
+
+ gguf_format_config = {}
+ for layer_name, config in layer_config.items():
+ if config["bits"] >= 16:
+ continue
+ gguf_format_config[layer_name] = gguf_format
+ i_layer = _get_digital_in_layer_name(layer_name)
+ if i_layer is None:
+ continue
+ gguf_name = tensor_map.get_name(layer_name)
+ if gguf_name is None:
+ continue
+ # attn_v
+ if "attn_v" in gguf_name:
+ if _use_more_bits(i_layer, n_layer):
+ # q6_k
+ config = _set_config(config, GGUF_CONFIG["gguf:q6_k"])
+ gguf_format_config[layer_name] = "gguf:q6_k"
+
+ # ffn_down
+ if "ffn_down" in gguf_name:
+ if gguf.MODEL_ARCH.FALCON == model_class.model_arch:
+ if i_layer < n_layer // 16:
+ format = "gguf:q6_k"
+ elif _use_more_bits(i_layer, n_layer):
+ format = "gguf:q5_k_s"
+ else:
+ format = "gguf:q4_k_s"
+ config = _set_config(config, GGUF_CONFIG[format])
+ gguf_format_config[layer_name] = format
+ else:
+ if _use_more_bits(i_layer, n_layer):
+ config = _set_config(config, GGUF_CONFIG["gguf:q6_k"])
+ gguf_format_config[layer_name] = "gguf:q6_k"
+
+ # attn_output
+ if "attn_output" in gguf_name and gguf.MODEL_ARCH.FALCON != model_class.model_arch:
+ n_export = 0
+ for name in ["num_experts", "num_local_experts", "n_routed_experts"]:
+ if hasattr(model.config, name):
+ n_export = getattr(model.config, name)
+ if n_export == 8:
+ config = _set_config(config, GGUF_CONFIG["gguf:q5_k_s"])
+ gguf_format_config[layer_name] = "gguf:q5_k_s"
+
+ # attn_qkv
+ if "attn_qkv" in gguf_name:
+ config = _set_config(config, GGUF_CONFIG["gguf:q5_k_s"])
+
+ layer_config[layer_name] = config
+ return layer_config, gguf_format_config
\ No newline at end of file
diff --git a/test/test_cuda/test_gguf.py b/test/test_cuda/test_gguf.py
index 430446726..022508213 100644
--- a/test/test_cuda/test_gguf.py
+++ b/test/test_cuda/test_gguf.py
@@ -170,6 +170,34 @@ class TestAutoRound(unittest.TestCase):
result = simple_evaluate_user_model(model, self.tokenizer, batch_size=16, tasks="piqa")
self.assertGreater(result['results']['piqa']['acc,none'], 0.55)
shutil.rmtree("./saved", ignore_errors=True)
+
+ @require_gguf
+ def test_q4_k_m(self):
+ model_name = "/models/Qwen2.5-1.5B-Instruct"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ bits, group_size, sym, super_group_size, super_bits = 4, 32, False, 8, 6
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=0,
+ seqlen=1,
+ dataset=self.llm_dataloader,
+ data_type="int_asym_dq",
+ super_group_size=super_group_size,
+ super_bits=super_bits
+ )
+ quantized_model_path = "./saved"
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="gguf:q4_k_m")
+ self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["super_group_size"], 16)
+ self.assertEqual(autoround.layer_config["model.layers.11.self_attn.v_proj"]["data_type"], "gguf_int_sym_dq")
+ self.assertEqual(autoround.layer_config["model.layers.7.self_attn.v_proj"]["data_type"], "gguf_int_asym_dq")
+ self.assertEqual(autoround.model.model.layers[0].self_attn.v_proj.bits, 6)
+ self.assertEqual(autoround.model.model.layers[12].self_attn.v_proj.bits, 4)
+ shutil.rmtree("./saved", ignore_errors=True)
if __name__ == "__main__":
unittest.main()
\ No newline at end of file