PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
rename backend to packing_format in config.json (#521)
Wenhua Cheng471dae81a9bd
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 102dcae3d..6f8401a7d 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -261,10 +261,9 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:exlayer_config = kwargs["layer_config"]quantization_config = kwargs["serialization_dict"]quantization_config["quant_method"] = "auto-round"-- quantization_config["backend"] = backendif quantization_config["bits"] == 3:backend = "auto_round:auto_gptq"+ quantization_config["packing_format"] = backendtokenizer = kwargs.get("tokenizer", None)processor = kwargs.get("processor", None)diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.pyindex cc78965ad..fa66a2862 100644--- a/auto_round/inference/auto_quantizer.py+++ b/auto_round/inference/auto_quantizer.py@@ -239,7 +239,7 @@ class AutoRoundConfig(QuantizationConfigMixin):self.dataset = datasetself.group_size = group_sizeself.sym = sym- self.target_backend = "auto"+ self.packing_format = "auto_round:auto_gptq"self.backend = backendself.layer_config = layer_configif kwargs is not None:@@ -256,7 +256,7 @@ class AutoRoundConfig(QuantizationConfigMixin):raise ValueError("group_size must be greater than 0 or equal to -1")def get_loading_attributes(self):- loading_attibutes_dict = {"target_backend": self.backend}+ loading_attibutes_dict = {"backend": self.backend}return loading_attibutes_dictdef to_dict(self):@@ -280,7 +280,7 @@ class AutoRoundConfig(QuantizationConfigMixin):"Failed to convert awq format to auto_round format. Only supports awq format with gemm version")if "auto-round" not in quant_method:- config_dict["backend"] = f"auto_round:{quant_method}"+ config_dict["packing_format"] = f"auto_round:{quant_method}"return super().from_dict(config_dict, return_unused_kwargs=return_unused_kwargs, **kwargs)diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.pyindex a29256157..fe26dd3f9 100644--- a/auto_round/inference/backend.py+++ b/auto_round/inference/backend.py@@ -104,7 +104,8 @@ BackendInfos['auto_gptq:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, Fagroup_size=[-1, 32, 64, 128, 256, 384, 512, 1024, 2048],##16 seems has accuracy issuefeature_checks=[exllamav2_feature_check],- alias=['gptq', 'auto_gptq', 'exllamav2', "gptq:exllamav2"],+ alias=['gptq', 'auto_gptq', 'exllamav2', "gptq:exllamav2",+ "auto_gptq:exllamav2"],requirements=["auto-gptq>=0.7.1"])diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.pyindex 417276939..e3aa9717e 100644--- a/auto_round/inference/convert_model.py+++ b/auto_round/inference/convert_model.py@@ -33,6 +33,7 @@ logger = getLogger(__name__)supported_devices = ("cpu", "hpu", "xpu", "cuda")+def flatten_list(nested_list):flattened = []for item in nested_list:@@ -42,9 +43,10 @@ def flatten_list(nested_list):flattened.append(item)return flattened+def skip_not_convert_modules(model, quantization_config, layer_names, layer_configs):modules_to_not_convert = getattr(quantization_config, "modules_to_not_convert", [])- try: # transformers new api+ try: # transformers new apimodules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert, add_default_skips=True)except Exception:modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert)@@ -106,13 +108,13 @@ def get_keys_to_not_convert(model):return filtered_module_names+def _get_modules_to_not_convert(model,- skip_modules = None,- keep_in_fp32_modules = None,+ skip_modules=None,+ keep_in_fp32_modules=None,add_default_skips: bool = False,- ):-+):if skip_modules is None or add_default_skips:modules_to_not_convert = get_keys_to_not_convert(model)else:@@ -126,12 +128,15 @@ def _get_modules_to_not_convert(return modules_to_not_convert+try:from transformers.quantizers.base import HfQuantizer+get_modules_to_not_convert = HfQuantizer.get_modules_to_not_convertexcept:get_modules_to_not_convert = _get_modules_to_not_convert+def get_available_devices():"""Returns a list of available devices in the current environment.@@ -310,9 +315,9 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, trequirements_info = process_requirement(requirements)if len(requirements_info) > 0:extra_info = ""- for index,req in enumerate(requirements_info):+ for index, req in enumerate(requirements_info):extra_info += (f"`{req}`")- if index != len(requirements_info)-1:+ if index != len(requirements_info) - 1:extra_info += " and "raise ImportError(f"{target_backend} requires the follow libraries. Please install them via {extra_info}")@@ -472,7 +477,7 @@ def post_init(model, used_backends):for l in data_types[1:]:common &= set(l)common = list(common)- if len(common)>0 and str(model.dtype).split('.')[-1] not in common:+ if len(common) > 0 and str(model.dtype).split('.')[-1] not in common:if common[0] == "float16":model = model.to(torch.float16)logger.warning("force model to float16")@@ -510,49 +515,51 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):raise NotImplementedError("This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound")- if not hasattr(quantization_config, "target_backend"):- quantization_config.target_backend = quantization_config.backend+ if hasattr(quantization_config, "backend"):+ backend = quantization_config.backend+ else:+ backend = "auto"+##target_backend could be None- _, target_backend = parse_target_device_and_backend(quantization_config.target_backend)+ _, backend = parse_target_device_and_backend(backend)- if hasattr(quantization_config, "backend"): # pragma: no cover- backend = quantization_config.backend+ if hasattr(quantization_config, "packing_format"): # pragma: no cover+ packing_format = quantization_config.packing_formatelif 'gptq' in quantization_config.quant_method: # pragma: no cover- backend = "auto_gptq"+ packing_format = "auto_gptq"elif "awq" in quantization_config.quant_method:- backend = "auto_awq"+ packing_format = "auto_awq"else: # pragma: no cover- backend = "auto_gptq"+ packing_format = "auto_gptq"logger.warning("Quantization backend must be specified. Set it to 'auto_gptq' by default.")- if backend == "auto":- backend = "auto_gptq"+ if packing_format == "auto":+ packing_format = "auto_gptq"layer_configs = get_layer_config(model, quantization_config)- if backend.startswith("auto_round:") and ("gptq" in backend or "awq" in backend):- backend = backend[len("auto_round:"):]- backend = find_backend(backend)+ if packing_format.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):+ packing_format = packing_format[len("auto_round:"):]+ orig_backend = find_backend(packing_format)- if target_backend.startswith("auto_round:") and ("gptq" in backend or "awq" in backend):- target_backend = target_backend[len("auto_round:"):]+ if backend.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):+ backend = backend[len("auto_round:"):]- used_backends = _replace_by_quant_layers(model, layer_configs, target_backend, target_device, backend)+ used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, orig_backend)- if target_backend == "auto" or target_backend == "":+ if backend == "auto" or backend == "":best_backend = get_highest_priority_backend(quantization_config.bits, quantization_config.sym,quantization_config.group_size, target_device,- BackendInfos[backend].packing_format)+ BackendInfos[orig_backend].packing_format)if best_backend is not None and best_backend not in used_backends:info = f"better backend is found, please install all the following requirements to enable it,"extra_info = inforequirements = BackendInfos[best_backend].requirementsrequirements_info = process_requirement(requirements)- for index,req in enumerate(requirements_info):+ for index, req in enumerate(requirements_info):extra_info += (f"`{req}`")if index != len(requirements_info) - 1:extra_info += " and "-logger.warning(extra_info)return model, used_backendsdiff --git a/test/test_autoround.py b/test/test_autoround.pyindex 052659aaa..77ad0dacc 100644--- a/test/test_autoround.py+++ b/test/test_autoround.py@@ -13,6 +13,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizerfrom auto_round import AutoRoundfrom _test_helpers import model_infer+class LLMDataLoader:def __init__(self):self.batch_size = 1@@ -324,6 +325,50 @@ class TestAutoRound(unittest.TestCase):)autoround.quantize()+ def test_tensor_reshape(self):+ model_name = "facebook/opt-125m"+ bits, group_size, sym = 4, 100, False+ autoround = AutoRound(+ self.model,+ self.tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=2,+ seqlen=2,+ dataset=self.llm_dataloader,+ )+ autoround.quantize()++ def test_rtn(self):+ model_name = "facebook/opt-125m"+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)++ bits, group_size, sym = 4, 128, True+ autoround = AutoRound(+ model,+ tokenizer,+ bits=bits,+ group_size=group_size,+ sym=sym,+ iters=1,+ nsamples=1++ )+ quantized_model_path = self.save_folder+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")+ from auto_round import AutoRoundConfig+ model = AutoModelForCausalLM.from_pretrained(+ self.save_folder,+ torch_dtype=torch.float16,+ device_map="auto",+ )++ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)+ model_infer(model, tokenizer)+ shutil.rmtree(self.save_folder)+def test_fallback_layers(self):bits, group_size, sym = 4, 128, Truemodel_name = "facebook/opt-125m"@@ -348,7 +393,7 @@ class TestAutoRound(unittest.TestCase):autoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)from auto_round import AutoRoundConfigquantization_config = AutoRoundConfig(- backend="cpu:auto_round:qbits_zp"+ backend="ipex")model = AutoModelForCausalLM.from_pretrained(quantized_model_path,@@ -359,24 +404,6 @@ class TestAutoRound(unittest.TestCase):res = tokenizer.decode(model.generate(**inputs, max_new_tokens=1)[0])shutil.rmtree(self.save_folder, ignore_errors=True)-- def test_tensor_reshape(self):- model_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True,- device_map='auto')- bits, group_size, sym = 4, 100, False- autoround = AutoRound(- self.model,- self.tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=2,- seqlen=2,- dataset=self.llm_dataloader,- )- autoround.quantize()-def test_not_convert_modules(self):import requestsfrom PIL import Image@@ -386,7 +413,7 @@ class TestAutoRound(unittest.TestCase):model_name = "Qwen/Qwen2-VL-2B-Instruct-AWQ"quantization_config = AutoRoundConfig()model = Qwen2VLForConditionalGeneration.from_pretrained(- model_name, quantization_config=quantization_config, torch_dtype=torch.float16)+ model_name, quantization_config=quantization_config, device_map="cpu", torch_dtype=torch.float16)self.assertTrue(isinstance(model.visual.blocks[0].attn.qkv, torch.nn.Linear))self.assertFalse(isinstance(model.visual.merger.mlp[0], QuantLinear))self.assertTrue(isinstance(model.model.layers[0].self_attn.v_proj, QuantLinear))@@ -428,35 +455,6 @@ class TestAutoRound(unittest.TestCase):)print(output_text)- def test_rtn(self):- model_name = "facebook/opt-125m"- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)-- bits, group_size, sym = 4, 128, True- autoround = AutoRound(- model,- tokenizer,- bits=bits,- group_size=group_size,- sym=sym,- iters=1,- nsamples=1-- )- quantized_model_path = self.save_folder- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")- from auto_round import AutoRoundConfig- model = AutoModelForCausalLM.from_pretrained(- self.save_folder,- torch_dtype=torch.float16,- device_map="auto",- )-- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)- model_infer(model, tokenizer)- shutil.rmtree(self.save_folder)-def test_fallback_layers_regex_awq(self):model_name = "facebook/opt-125m"bits, group_size, sym = 4, 128, True@@ -483,7 +481,7 @@ class TestAutoRound(unittest.TestCase):quantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map='auto',quantization_config=quantization_config)+ device_map='auto', quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -518,7 +516,7 @@ class TestAutoRound(unittest.TestCase):quantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map='auto',quantization_config=quantization_config)+ device_map='auto', quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -553,7 +551,7 @@ class TestAutoRound(unittest.TestCase):quantization_config = AutoRoundConfig()model = AutoModelForCausalLM.from_pretrained(quantized_model_path,- device_map='auto',quantization_config=quantization_config)+ device_map='auto', quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)@@ -561,15 +559,14 @@ class TestAutoRound(unittest.TestCase):print(res)shutil.rmtree(self.save_folder, ignore_errors=True)-def test_fallback_layers_regex_exception(self):model_name = "facebook/opt-125m"bits, group_size, sym = 4, 128, Truemodel = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)layer_config = {- "model.decoder.layers.12.self_attn.k_proj": {"bits": 16}- }+ "model.decoder.layers.12.self_attn.k_proj": {"bits": 16}+ }with self.assertRaises(ValueError):autoround = AutoRound(model,@@ -584,8 +581,5 @@ class TestAutoRound(unittest.TestCase):)---if __name__ == "__main__":unittest.main()