Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

rename backend to packing_format in config.json (#521)

Wenhua Cheng471dae81a9bd
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 102dcae3d..6f8401a7d 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -261,10 +261,9 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
layer_config = kwargs["layer_config"]
quantization_config = kwargs["serialization_dict"]
quantization_config["quant_method"] = "auto-round"
-
- quantization_config["backend"] = backend
if quantization_config["bits"] == 3:
backend = "auto_round:auto_gptq"
+ quantization_config["packing_format"] = backend
tokenizer = kwargs.get("tokenizer", None)
processor = kwargs.get("processor", None)
diff --git a/auto_round/inference/auto_quantizer.py b/auto_round/inference/auto_quantizer.py
index cc78965ad..fa66a2862 100644
--- a/auto_round/inference/auto_quantizer.py
+++ b/auto_round/inference/auto_quantizer.py
@@ -239,7 +239,7 @@ class AutoRoundConfig(QuantizationConfigMixin):
self.dataset = dataset
self.group_size = group_size
self.sym = sym
- self.target_backend = "auto"
+ self.packing_format = "auto_round:auto_gptq"
self.backend = backend
self.layer_config = layer_config
if kwargs is not None:
@@ -256,7 +256,7 @@ class AutoRoundConfig(QuantizationConfigMixin):
raise ValueError("group_size must be greater than 0 or equal to -1")
def get_loading_attributes(self):
- loading_attibutes_dict = {"target_backend": self.backend}
+ loading_attibutes_dict = {"backend": self.backend}
return loading_attibutes_dict
def to_dict(self):
@@ -280,7 +280,7 @@ class AutoRoundConfig(QuantizationConfigMixin):
"Failed to convert awq format to auto_round format. Only supports awq format with gemm version")
if "auto-round" not in quant_method:
- config_dict["backend"] = f"auto_round:{quant_method}"
+ config_dict["packing_format"] = f"auto_round:{quant_method}"
return super().from_dict(config_dict, return_unused_kwargs=return_unused_kwargs, **kwargs)
diff --git a/auto_round/inference/backend.py b/auto_round/inference/backend.py
index a29256157..fe26dd3f9 100644
--- a/auto_round/inference/backend.py
+++ b/auto_round/inference/backend.py
@@ -104,7 +104,8 @@ BackendInfos['auto_gptq:exllamav2'] = BackendInfo(device=["cuda"], sym=[True, Fa
group_size=[-1, 32, 64, 128, 256, 384, 512, 1024, 2048],
##16 seems has accuracy issue
feature_checks=[exllamav2_feature_check],
- alias=['gptq', 'auto_gptq', 'exllamav2', "gptq:exllamav2"],
+ alias=['gptq', 'auto_gptq', 'exllamav2', "gptq:exllamav2",
+ "auto_gptq:exllamav2"],
requirements=["auto-gptq>=0.7.1"]
)
diff --git a/auto_round/inference/convert_model.py b/auto_round/inference/convert_model.py
index 417276939..e3aa9717e 100644
--- a/auto_round/inference/convert_model.py
+++ b/auto_round/inference/convert_model.py
@@ -33,6 +33,7 @@ logger = getLogger(__name__)
supported_devices = ("cpu", "hpu", "xpu", "cuda")
+
def flatten_list(nested_list):
flattened = []
for item in nested_list:
@@ -42,9 +43,10 @@ def flatten_list(nested_list):
flattened.append(item)
return flattened
+
def skip_not_convert_modules(model, quantization_config, layer_names, layer_configs):
modules_to_not_convert = getattr(quantization_config, "modules_to_not_convert", [])
- try: # transformers new api
+ try: # transformers new api
modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert, add_default_skips=True)
except Exception:
modules_to_not_convert = get_modules_to_not_convert(model, modules_to_not_convert)
@@ -106,13 +108,13 @@ def get_keys_to_not_convert(model):
return filtered_module_names
+
def _get_modules_to_not_convert(
model,
- skip_modules = None,
- keep_in_fp32_modules = None,
+ skip_modules=None,
+ keep_in_fp32_modules=None,
add_default_skips: bool = False,
- ):
-
+):
if skip_modules is None or add_default_skips:
modules_to_not_convert = get_keys_to_not_convert(model)
else:
@@ -126,12 +128,15 @@ def _get_modules_to_not_convert(
return modules_to_not_convert
+
try:
from transformers.quantizers.base import HfQuantizer
+
get_modules_to_not_convert = HfQuantizer.get_modules_to_not_convert
except:
get_modules_to_not_convert = _get_modules_to_not_convert
+
def get_available_devices():
"""
Returns a list of available devices in the current environment.
@@ -310,9 +315,9 @@ def _replace_by_quant_layers(module: nn.Module, layer_configs, target_backend, t
requirements_info = process_requirement(requirements)
if len(requirements_info) > 0:
extra_info = ""
- for index,req in enumerate(requirements_info):
+ for index, req in enumerate(requirements_info):
extra_info += (f"`{req}`")
- if index != len(requirements_info)-1:
+ if index != len(requirements_info) - 1:
extra_info += " and "
raise ImportError(f"{target_backend} requires the follow libraries. Please install them via {extra_info}")
@@ -472,7 +477,7 @@ def post_init(model, used_backends):
for l in data_types[1:]:
common &= set(l)
common = list(common)
- if len(common)>0 and str(model.dtype).split('.')[-1] not in common:
+ if len(common) > 0 and str(model.dtype).split('.')[-1] not in common:
if common[0] == "float16":
model = model.to(torch.float16)
logger.warning("force model to float16")
@@ -510,49 +515,51 @@ def convert_hf_model(model: nn.Module, target_device="cpu"):
raise NotImplementedError(
"This GPTQ model may contain a non-dummy g_idx, which is not yet supported by AutoRound")
- if not hasattr(quantization_config, "target_backend"):
- quantization_config.target_backend = quantization_config.backend
+ if hasattr(quantization_config, "backend"):
+ backend = quantization_config.backend
+ else:
+ backend = "auto"
+
##target_backend could be None
- _, target_backend = parse_target_device_and_backend(quantization_config.target_backend)
+ _, backend = parse_target_device_and_backend(backend)
- if hasattr(quantization_config, "backend"): # pragma: no cover
- backend = quantization_config.backend
+ if hasattr(quantization_config, "packing_format"): # pragma: no cover
+ packing_format = quantization_config.packing_format
elif 'gptq' in quantization_config.quant_method: # pragma: no cover
- backend = "auto_gptq"
+ packing_format = "auto_gptq"
elif "awq" in quantization_config.quant_method:
- backend = "auto_awq"
+ packing_format = "auto_awq"
else: # pragma: no cover
- backend = "auto_gptq"
+ packing_format = "auto_gptq"
logger.warning("Quantization backend must be specified. Set it to 'auto_gptq' by default.")
- if backend == "auto":
- backend = "auto_gptq"
+ if packing_format == "auto":
+ packing_format = "auto_gptq"
layer_configs = get_layer_config(model, quantization_config)
- if backend.startswith("auto_round:") and ("gptq" in backend or "awq" in backend):
- backend = backend[len("auto_round:"):]
- backend = find_backend(backend)
+ if packing_format.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):
+ packing_format = packing_format[len("auto_round:"):]
+ orig_backend = find_backend(packing_format)
- if target_backend.startswith("auto_round:") and ("gptq" in backend or "awq" in backend):
- target_backend = target_backend[len("auto_round:"):]
+ if backend.startswith("auto_round:") and ("gptq" in packing_format or "awq" in packing_format):
+ backend = backend[len("auto_round:"):]
- used_backends = _replace_by_quant_layers(model, layer_configs, target_backend, target_device, backend)
+ used_backends = _replace_by_quant_layers(model, layer_configs, backend, target_device, orig_backend)
- if target_backend == "auto" or target_backend == "":
+ if backend == "auto" or backend == "":
best_backend = get_highest_priority_backend(quantization_config.bits, quantization_config.sym,
quantization_config.group_size, target_device,
- BackendInfos[backend].packing_format)
+ BackendInfos[orig_backend].packing_format)
if best_backend is not None and best_backend not in used_backends:
info = f"better backend is found, please install all the following requirements to enable it,"
extra_info = info
requirements = BackendInfos[best_backend].requirements
requirements_info = process_requirement(requirements)
- for index,req in enumerate(requirements_info):
+ for index, req in enumerate(requirements_info):
extra_info += (f"`{req}`")
if index != len(requirements_info) - 1:
extra_info += " and "
-
logger.warning(extra_info)
return model, used_backends
diff --git a/test/test_autoround.py b/test/test_autoround.py
index 052659aaa..77ad0dacc 100644
--- a/test/test_autoround.py
+++ b/test/test_autoround.py
@@ -13,6 +13,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
from _test_helpers import model_infer
+
class LLMDataLoader:
def __init__(self):
self.batch_size = 1
@@ -324,6 +325,50 @@ class TestAutoRound(unittest.TestCase):
)
autoround.quantize()
+ def test_tensor_reshape(self):
+ model_name = "facebook/opt-125m"
+ bits, group_size, sym = 4, 100, False
+ autoround = AutoRound(
+ self.model,
+ self.tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=2,
+ seqlen=2,
+ dataset=self.llm_dataloader,
+ )
+ autoround.quantize()
+
+ def test_rtn(self):
+ model_name = "facebook/opt-125m"
+ model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+
+ bits, group_size, sym = 4, 128, True
+ autoround = AutoRound(
+ model,
+ tokenizer,
+ bits=bits,
+ group_size=group_size,
+ sym=sym,
+ iters=1,
+ nsamples=1
+
+ )
+ quantized_model_path = self.save_folder
+ autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
+ from auto_round import AutoRoundConfig
+ model = AutoModelForCausalLM.from_pretrained(
+ self.save_folder,
+ torch_dtype=torch.float16,
+ device_map="auto",
+ )
+
+ tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
+ model_infer(model, tokenizer)
+ shutil.rmtree(self.save_folder)
+
def test_fallback_layers(self):
bits, group_size, sym = 4, 128, True
model_name = "facebook/opt-125m"
@@ -348,7 +393,7 @@ class TestAutoRound(unittest.TestCase):
autoround.save_quantized(output_dir=quantized_model_path, format="auto_round", inplace=True)
from auto_round import AutoRoundConfig
quantization_config = AutoRoundConfig(
- backend="cpu:auto_round:qbits_zp"
+ backend="ipex"
)
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
@@ -359,24 +404,6 @@ class TestAutoRound(unittest.TestCase):
res = tokenizer.decode(model.generate(**inputs, max_new_tokens=1)[0])
shutil.rmtree(self.save_folder, ignore_errors=True)
-
- def test_tensor_reshape(self):
- model_name = "facebook/opt-125m"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float32, trust_remote_code=True,
- device_map='auto')
- bits, group_size, sym = 4, 100, False
- autoround = AutoRound(
- self.model,
- self.tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=2,
- seqlen=2,
- dataset=self.llm_dataloader,
- )
- autoround.quantize()
-
def test_not_convert_modules(self):
import requests
from PIL import Image
@@ -386,7 +413,7 @@ class TestAutoRound(unittest.TestCase):
model_name = "Qwen/Qwen2-VL-2B-Instruct-AWQ"
quantization_config = AutoRoundConfig()
model = Qwen2VLForConditionalGeneration.from_pretrained(
- model_name, quantization_config=quantization_config, torch_dtype=torch.float16)
+ model_name, quantization_config=quantization_config, device_map="cpu", torch_dtype=torch.float16)
self.assertTrue(isinstance(model.visual.blocks[0].attn.qkv, torch.nn.Linear))
self.assertFalse(isinstance(model.visual.merger.mlp[0], QuantLinear))
self.assertTrue(isinstance(model.model.layers[0].self_attn.v_proj, QuantLinear))
@@ -428,35 +455,6 @@ class TestAutoRound(unittest.TestCase):
)
print(output_text)
- def test_rtn(self):
- model_name = "facebook/opt-125m"
- model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-
- bits, group_size, sym = 4, 128, True
- autoround = AutoRound(
- model,
- tokenizer,
- bits=bits,
- group_size=group_size,
- sym=sym,
- iters=1,
- nsamples=1
-
- )
- quantized_model_path = self.save_folder
- autoround.quantize_and_save(output_dir=quantized_model_path, format="auto_round")
- from auto_round import AutoRoundConfig
- model = AutoModelForCausalLM.from_pretrained(
- self.save_folder,
- torch_dtype=torch.float16,
- device_map="auto",
- )
-
- tokenizer = AutoTokenizer.from_pretrained(self.save_folder)
- model_infer(model, tokenizer)
- shutil.rmtree(self.save_folder)
-
def test_fallback_layers_regex_awq(self):
model_name = "facebook/opt-125m"
bits, group_size, sym = 4, 128, True
@@ -483,7 +481,7 @@ class TestAutoRound(unittest.TestCase):
quantization_config = AutoRoundConfig()
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map='auto',quantization_config=quantization_config)
+ device_map='auto', quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
@@ -518,7 +516,7 @@ class TestAutoRound(unittest.TestCase):
quantization_config = AutoRoundConfig()
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map='auto',quantization_config=quantization_config)
+ device_map='auto', quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
@@ -553,7 +551,7 @@ class TestAutoRound(unittest.TestCase):
quantization_config = AutoRoundConfig()
model = AutoModelForCausalLM.from_pretrained(quantized_model_path,
- device_map='auto',quantization_config=quantization_config)
+ device_map='auto', quantization_config=quantization_config)
tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)
text = "There is a girl who likes adventure,"
inputs = tokenizer(text, return_tensors="pt").to(model.device)
@@ -561,15 +559,14 @@ class TestAutoRound(unittest.TestCase):
print(res)
shutil.rmtree(self.save_folder, ignore_errors=True)
-
def test_fallback_layers_regex_exception(self):
model_name = "facebook/opt-125m"
bits, group_size, sym = 4, 128, True
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto", trust_remote_code=True)
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
layer_config = {
- "model.decoder.layers.12.self_attn.k_proj": {"bits": 16}
- }
+ "model.decoder.layers.12.self_attn.k_proj": {"bits": 16}
+ }
with self.assertRaises(ValueError):
autoround = AutoRound(
model,
@@ -584,8 +581,5 @@ class TestAutoRound(unittest.TestCase):
)
-
-
-
if __name__ == "__main__":
unittest.main()