PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
add regex support in layer_config setting (#519)
Wenhua Chengd286538f55d3
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex af175fa18..a75e0349c 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -691,19 +691,43 @@ class AutoRound(object):# Get the names of layers in quantization blockslayers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)+ ##process regex in layer_config+ all_supported_layer_names = []# List of configuration keyskeys = self.serialization_keys- has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)-- # Iterate through all modules in the modelfor n, m in self.model.named_modules():-# Delete previous configuration to avoid conflicts with prior tuningfor key in keys:if hasattr(m, key):delattr(m, key)+ # Skip unsupported types+ if not isinstance(m, tuple(self.supported_types)):+ continue+ all_supported_layer_names.append(n)++ names_in_layer_config = list(layer_config.keys())+ for name in names_in_layer_config:+ if name in all_supported_layer_names:+ continue+ matched_names = []+ for layer_name in all_supported_layer_names:+ if re.search(re.compile(name), layer_name) is not None:+ matched_names.append(layer_name)+ if len(matched_names) > 0:+ val = layer_config[name]+ layer_config.pop(name)+ for match_name in matched_names:+ layer_config[match_name] = val+ else:+ raise ValueError(f"key {name} in layer_config is invalid, please have a double check")++ has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)++ # Iterate through all modules in the model+ for n, m in self.model.named_modules():+# Skip unsupported typesif not isinstance(m, tuple(self.supported_types)):continuediff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.pyindex 39345cc3e..65f9088bf 100644--- a/auto_round/export/export_to_autogptq/export.py+++ b/auto_round/export/export_to_autogptq/export.py@@ -60,7 +60,6 @@ BLOCK_PATTERNS = [ ## copy from transformers optimumdef pack_layer(name, model, backend):-if name == "lm_head": ##dese not support lm-headreturnlayer = get_module(model, name)@@ -126,11 +125,11 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllquant_block_list = kwargs.get("quant_block_list", get_block_names(model))tokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)- if os.path.exists(output_dir):+ if output_dir is not None and os.path.exists(output_dir):logger.warning(f"{output_dir} already exists, this may cause model conflict")- if tokenizer is not None:+ if output_dir is not None and tokenizer is not None:tokenizer.save_pretrained(output_dir)- if processor is not None:+ if output_dir is not None and processor is not None:processor.save_pretrained(output_dir)##check module quantized in block, this may have bug for mixed precision quantizationquantization_config = kwargs["serialization_dict"]@@ -144,8 +143,8 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll## as layers maybe already packed, we need to check in layer_configlayer_config = kwargs["layer_config"]- for n,m in model.named_modules():- m.tmp_name=n+ for n, m in model.named_modules():+ m.tmp_name = nall_to_quantized = Truemodules_in_block_to_quantize = []@@ -163,7 +162,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exllmodules_in_block_to_quantize = Nonefor n, m in model.named_modules():- delattr(m,"tmp_name")+ delattr(m, "tmp_name")if not inplace:model = copy.deepcopy(model.to("cpu"))diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.pyindex a55c4f863..6cfe53778 100644--- a/auto_round/export/export_to_awq/export.py+++ b/auto_round/export/export_to_awq/export.py@@ -78,10 +78,13 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):processor = kwargs.get("processor", None)modules_to_not_convert = []+ if output_dir is not None and os.path.exists(output_dir):+ logger.warning(f"{output_dir} already exists, this may cause model conflict")+logger.info("Saving quantized model to auto_awq format")- if tokenizer is not None:+ if tokenizer is not None and output_dir is not None:tokenizer.save_pretrained(output_dir)- if processor is not None:+ if processor is not None and output_dir is not None:processor.save_pretrained(output_dir)# mllm modelsall_blocks = get_block_names(model, quant_vision=True)@@ -119,8 +122,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):if output_dir is None:return compressed_model- if os.path.exists(output_dir):- logger.warning(f"{output_dir} already exists, this may cause model conflict")+layer_config = kwargs["layer_config"]for key in layer_config.keys():if not check_to_quantized(layer_config[key]) and \diff --git a/test/test_autoround.py b/test/test_autoround.pyindex 58105e973..052659aaa 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -457,6 +457,135 @@ class TestAutoRound(unittest.TestCase):model_infer(model, tokenizer)shutil.rmtree(self.save_folder)+ def test_fallback_layers_regex_awq(self):+ model_name = "facebook/opt-125m"+ bits, group_size, sym = 4, 128, True+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ layer_config = {"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},+ "model.decoder.layers.1.self_attn.k_proj": {"bits": 16}}+ autoround = AutoRound(+ model,+ tokenizer=tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config+ )+ autoround.quantize()+ quantized_model_path = self.save_folder++ autoround.save_quantized(output_dir=quantized_model_path, format="auto_awq", inplace=True)+ from auto_round import AutoRoundConfig+ quantization_config = AutoRoundConfig()++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,+ device_map='auto',quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])+ print(res)+ shutil.rmtree(self.save_folder, ignore_errors=True)++ def test_fallback_layers_regex_gptq(self):+ model_name = "facebook/opt-125m"+ bits, group_size, sym = 4, 128, True+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ layer_config = {"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},+ ##"model.decoder.layers.1.self_attn.k_proj": {"bits": 16}+ }+ autoround = AutoRound(+ model,+ tokenizer=tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config+ )+ autoround.quantize()+ quantized_model_path = self.save_folder++ autoround.save_quantized(output_dir=quantized_model_path, format="auto_gptq", inplace=True)+ from auto_round import AutoRoundConfig+ quantization_config = AutoRoundConfig()++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,+ device_map='auto',quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])+ print(res)+ shutil.rmtree(self.save_folder, ignore_errors=True)++ def test_fallback_layers_regex_round(self):+ model_name = "facebook/opt-125m"+ bits, group_size, sym = 4, 128, True+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ layer_config = {"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},+ "model.decoder.layers.1.self_attn.k_proj": {"bits": 16}+ }+ autoround = AutoRound(+ model,+ tokenizer=tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config+ )+ autoround.quantize()+ quantized_model_path = self.save_folder++ autoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)+ from auto_round import AutoRoundConfig+ quantization_config = AutoRoundConfig()++ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,+ device_map='auto',quantization_config=quantization_config)+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)+ text = "There is a girl who likes adventure,"+ inputs = tokenizer(text, return_tensors="pt").to(model.device)+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])+ print(res)+ shutil.rmtree(self.save_folder, ignore_errors=True)+++ def test_fallback_layers_regex_exception(self):+ model_name = "facebook/opt-125m"+ bits, group_size, sym = 4, 128, True+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ layer_config = {+ "model.decoder.layers.12.self_attn.k_proj": {"bits": 16}+ }+ with self.assertRaises(ValueError):+ autoround = AutoRound(+ model,+ tokenizer=tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ layer_config=layer_config+ )++++if __name__ == "__main__":unittest.main()