Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

add regex support in layer_config setting (#519)

Wenhua Chengd286538f55d3
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index af175fa18..a75e0349c 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -691,19 +691,43 @@ class AutoRound(object):
# Get the names of layers in quantization blocks
layers_in_blocks = get_layer_names_in_block(self.model, self.supported_types, self.quant_block_list)
+ ##process regex in layer_config
+ all_supported_layer_names = []
# List of configuration keys
keys = self.serialization_keys
- has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)
-
- # Iterate through all modules in the model
for n, m in self.model.named_modules():
-
# Delete previous configuration to avoid conflicts with prior tuning
for key in keys:
if hasattr(m, key):
delattr(m, key)
+ # Skip unsupported types
+ if not isinstance(m, tuple(self.supported_types)):
+ continue
+ all_supported_layer_names.append(n)
+
+ names_in_layer_config = list(layer_config.keys())
+ for name in names_in_layer_config:
+ if name in all_supported_layer_names:
+ continue
+ matched_names = []
+ for layer_name in all_supported_layer_names:
+ if re.search(re.compile(name), layer_name) is not None:
+ matched_names.append(layer_name)
+ if len(matched_names) > 0:
+ val = layer_config[name]
+ layer_config.pop(name)
+ for match_name in matched_names:
+ layer_config[match_name] = val
+ else:
+ raise ValueError(f"key {name} in layer_config is invalid, please have a double check")
+
+ has_qlayer_outside_block = False # Flag to track if there are quantized layers outside blocks (e.g., lm-head)
+
+ # Iterate through all modules in the model
+ for n, m in self.model.named_modules():
+
# Skip unsupported types
if not isinstance(m, tuple(self.supported_types)):
continue
diff --git a/auto_round/export/export_to_autogptq/export.py b/auto_round/export/export_to_autogptq/export.py
index 39345cc3e..65f9088bf 100644
--- a/auto_round/export/export_to_autogptq/export.py
+++ b/auto_round/export/export_to_autogptq/export.py
@@ -60,7 +60,6 @@ BLOCK_PATTERNS = [ ## copy from transformers optimum
def pack_layer(name, model, backend):
-
if name == "lm_head": ##dese not support lm-head
return
layer = get_module(model, name)
@@ -126,11 +125,11 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
quant_block_list = kwargs.get("quant_block_list", get_block_names(model))
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
- if os.path.exists(output_dir):
+ if output_dir is not None and os.path.exists(output_dir):
logger.warning(f"{output_dir} already exists, this may cause model conflict")
- if tokenizer is not None:
+ if output_dir is not None and tokenizer is not None:
tokenizer.save_pretrained(output_dir)
- if processor is not None:
+ if output_dir is not None and processor is not None:
processor.save_pretrained(output_dir)
##check module quantized in block, this may have bug for mixed precision quantization
quantization_config = kwargs["serialization_dict"]
@@ -144,8 +143,8 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
## as layers maybe already packed, we need to check in layer_config
layer_config = kwargs["layer_config"]
- for n,m in model.named_modules():
- m.tmp_name=n
+ for n, m in model.named_modules():
+ m.tmp_name = n
all_to_quantized = True
modules_in_block_to_quantize = []
@@ -163,7 +162,7 @@ def save_quantized_as_autogptq(output_dir, inplace=True, backend="auto_gptq:exll
modules_in_block_to_quantize = None
for n, m in model.named_modules():
- delattr(m,"tmp_name")
+ delattr(m, "tmp_name")
if not inplace:
model = copy.deepcopy(model.to("cpu"))
diff --git a/auto_round/export/export_to_awq/export.py b/auto_round/export/export_to_awq/export.py
index a55c4f863..6cfe53778 100644
--- a/auto_round/export/export_to_awq/export.py
+++ b/auto_round/export/export_to_awq/export.py
@@ -78,10 +78,13 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
processor = kwargs.get("processor", None)
modules_to_not_convert = []
+ if output_dir is not None and os.path.exists(output_dir):
+ logger.warning(f"{output_dir} already exists, this may cause model conflict")
+
logger.info("Saving quantized model to auto_awq format")
- if tokenizer is not None:
+ if tokenizer is not None and output_dir is not None:
tokenizer.save_pretrained(output_dir)
- if processor is not None:
+ if processor is not None and output_dir is not None:
processor.save_pretrained(output_dir)
# mllm models
all_blocks = get_block_names(model, quant_vision=True)
@@ -119,8 +122,7 @@ def save_quantized_as_autoawq(output_dir, inplace=True, **kwargs):
if output_dir is None:
return compressed_model
- if os.path.exists(output_dir):
- logger.warning(f"{output_dir} already exists, this may cause model conflict")
+
layer_config = kwargs["layer_config"]
for key in layer_config.keys():
if not check_to_quantized(layer_config[key]) and \
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 58105e973..052659aaa 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -457,6 +457,135 @@ class TestAutoRound(unittest.TestCase):
model_infer(model, tokenizer)
shutil.rmtree(self.save_folder)
+ def test_fallback_layers_regex_awq(self):
+ model_name = "facebook/opt-125m"
+ bits, group_size, sym = 4, 128, True
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ layer_config = {"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},
+ "model.decoder.layers.1.self_attn.k_proj": {"bits": 16}}
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config
+ )
+ autoround.quantize()
+ quantized_model_path = self.save_folder
+
+ autoround.save_quantized(output_dir=quantized_model_path, format="auto_awq", inplace=True)
+ from auto_round import AutoRoundConfig
+ quantization_config = AutoRoundConfig()
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
+ device_map='auto',quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])
+ print(res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+ def test_fallback_layers_regex_gptq(self):
+ model_name = "facebook/opt-125m"
+ bits, group_size, sym = 4, 128, True
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ layer_config = {"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},
+ ##"model.decoder.layers.1.self_attn.k_proj": {"bits": 16}
+ }
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config
+ )
+ autoround.quantize()
+ quantized_model_path = self.save_folder
+
+ autoround.save_quantized(output_dir=quantized_model_path, format="auto_gptq", inplace=True)
+ from auto_round import AutoRoundConfig
+ quantization_config = AutoRoundConfig()
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
+ device_map='auto',quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])
+ print(res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+ def test_fallback_layers_regex_round(self):
+ model_name = "facebook/opt-125m"
+ bits, group_size, sym = 4, 128, True
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ layer_config = {"model\.decoder\.layers\.(?:[0-9]|1[0-1])\.self_attn\.q_proj": {"bits": 16},
+ "model.decoder.layers.1.self_attn.k_proj": {"bits": 16}
+ }
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config
+ )
+ autoround.quantize()
+ quantized_model_path = self.save_folder
+
+ autoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)
+ from auto_round import AutoRoundConfig
+ quantization_config = AutoRoundConfig()
+
+ model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
+ device_map='auto',quantization_config=quantization_config)
+ tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
+ text = "There is a girl who likes adventure,"
+ inputs = tokenizer(text, return_tensors="pt").to(model.device)
+ res = tokenizer.decode(model.generate(**inputs, max_new_tokens=5)[0])
+ print(res)
+ shutil.rmtree(self.save_folder, ignore_errors=True)
+
+
+ def test_fallback_layers_regex_exception(self):
+ model_name = "facebook/opt-125m"
+ bits, group_size, sym = 4, 128, True
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ layer_config = {
+ "model.decoder.layers.12.self_attn.k_proj": {"bits": 16}
+ }
+ with self.assertRaises(ValueError):
+ autoround = AutoRound(
+ model,
+ tokenizer=tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ layer_config=layer_config
+ )
+
+
+
+
if __name__ == "__main__":
unittest.main()