PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
set autoround format as default to unify CPU/HPU/CUDA (#205)
wenhuach211e75afd1df2f
diff --git a/README.md b/README.mdindex f14a08a79..b9f614aba 100644--- a/README.md+++ b/README.md@@ -55,9 +55,6 @@ pip install auto-round### Gaudi2/ CPU/ GPU-We found a significant accuracy discrepancy with the qdq model using the AutoGPTQ GPU backend with asymmetric-quantization in some scenarios, especially at lower bits,like 2. Please save quantized model to AuoRound format to fix this issue.-```pythonfrom transformers import AutoModelForCausalLM, AutoTokenizer@@ -71,7 +68,7 @@ bits, group_size, sym = 4, 128, Falseautoround = AutoRound(model, tokenizer, bits=bits, group_size=group_size, sym=sym)autoround.quantize()output_dir = "./tmp_autoround"-autoround.save_quantized(output_dir) ##save_quantized(output_dir,format="auto_round")+autoround.save_quantized(output_dir) ##save_quantized(output_dir,format="auto_gptq)```<details>@@ -135,49 +132,31 @@ autoround.save_quantized(output_dir) ##save_quantized(output_dir,format="auto_ro3 Setting 'minmax_lr' to 2.0/iters has been observed to occasionally yield improved results.## Model inference+Please run the quantization code first-Please run the quantization code first.+### Setup env for AutoRound format-### CPU-```python-##pip install intel-extension-for-transformers-from intel_extension_for_transformers.transformers import AutoModelForCausalLM-from transformers import AutoTokenizer+2 Set up the environment-quantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path)-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)-text = "There is a girl who likes adventure,"-inputs = tokenizer(text, return_tensors="pt").to(model.device)-print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))-```+**cuda**: git clone https://github.com/intel/auto-round.git && cd auto-round && pip install -vvv --no-build-isolation -e .-### GPU+cpu: pip install intel-extension-for-transformers-```python-##pip install auto-gptq-from transformers import AutoModelForCausalLM, AutoTokenizer-##from auto_round.auto_quantizer import AutoHfQuantizer ## uncomment it for models with auto_round format+hpu: docker image with Gaudi Software Stack is recommended, please refer to following script for environment setup. More details can be found in Gaudi Guide.-quantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map="auto")-tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)-text = "There is a girl who likes adventure,"-inputs = tokenizer(text, return_tensors="pt").to(model.device)-print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))-```-### Intel Gaudi-2+### Gaudi2/ CPU/ GPU```python-import torch-from transformers import AutoModelForCausalLM, AutoTokenizer-from auto_round.auto_quantizer import AutoHfQuantizer-import habana_frameworks.torch.core as htcore-import habana_frameworks.torch.hpu as hthpu+from transformers import AutoModelForCausalLM,AutoTokenizer+from auto_round import AutoRoundConfig+device = "auto" ##cpu, hpu, cuda+quantization_config = AutoRoundConfig(+ backend=device+)quantized_model_path = "./tmp_autoround"-model = AutoModelForCausalLM.from_pretrained(quantized_model_path).to('hpu').to(torch.float32)+model = AutoModelForCausalLM.from_pretrained(quantized_model_path, device_map=device,quantization_config=quantization_config)tokenizer = AutoTokenizer.from_pretrained(quantized_model_path)text = "There is a girl who likes adventure,"inputs = tokenizer(text, return_tensors="pt").to(model.device)diff --git a/auto_round/__init__.py b/auto_round/__init__.pyindex 46f42e4a2..0f00cf352 100644--- a/auto_round/__init__.py+++ b/auto_round/__init__.py@@ -12,4 +12,5 @@# See the License for the specific language governing permissions and# limitations under the License.from .autoround import AutoRound, AutoAdamRound, AutoOPTRound+from .auto_quantizer import AutoHfQuantizer,AutoRoundConfigfrom .version import __version__\ No newline at end of filediff --git a/auto_round/auto_quantizer.py b/auto_round/auto_quantizer.pyindex 4bc89710b..3aab37894 100644--- a/auto_round/auto_quantizer.py+++ b/auto_round/auto_quantizer.py@@ -44,6 +44,7 @@ from transformers.utils.quantization_config import AwqConfig, GPTQConfig, Quantifrom auto_round.utils import get_module, set_module, dynamic_import_inference_linearimport auto_round_extension.qbits.qlinear_qbits as qlinear_qbitsfrom enum import Enum+import copylogger = getLogger(__name__)import sys@@ -106,17 +107,19 @@ def is_autoround_exllamav2_available():res = Falsereturn res-def is_hpu_supported(): # pragma: no cover++def is_hpu_supported(): # pragma: no covertry:import subprocess- import habana_frameworks.torch.core as htcore # pylint: disable=E0401+ import habana_frameworks.torch.core as htcore # pylint: disable=E0401hqt_version = subprocess.check_output(['pip', 'show', \- 'habana_quantization_toolkit']).decode().split('\n')[1].split(': ')[1]- assert(hqt_version >= "1.17")+ 'habana_quantization_toolkit']).decode().split('\n')[1].split(': ')[1]+ assert (hqt_version >= "1.17")except ImportError as e:return Falsereturn True+if is_auto_round_available():from auto_round_extension.cuda.post_init import autoround_post_init@@ -155,7 +158,7 @@ class AutoHfQuantizer:f"Unknown quantization type, got {quant_method} - supported types are:"f" {list(AUTO_QUANTIZER_MAPPING.keys())}")- if "auto-round" in quant_method or is_hpu_supported(): # pragma: no cover+ if "auto-round" in quant_method or is_hpu_supported(): # pragma: no covertarget_cls = AutoRoundQuantizerelse:target_cls = AUTO_QUANTIZER_MAPPING[quant_method]@@ -190,7 +193,8 @@ class AutoHfQuantizer:else:quantization_config = AutoQuantizationConfig.from_dict(quantization_config) # pylint: disable=E1101- if isinstance(quantization_config, (GPTQConfig, AwqConfig)) and quantization_config_from_args is not None:+ if isinstance(quantization_config,+ (GPTQConfig, AwqConfig, AutoRoundConfig)) and quantization_config_from_args is not None:# special case for GPTQ / AWQ config collisionloading_attr_dict = quantization_config_from_args.get_loading_attributes()for attr, val in loading_attr_dict.items():@@ -228,12 +232,12 @@ class AutoRoundConfig(QuantizationConfigMixin):def __init__(self,- bits: int,+ bits: int = 4,tokenizer: Any = None,dataset: str = None,group_size: int = 128,sym: bool = False,- backend="auto_round:exllamav2",+ backend="auto",layer_config: dict = None,**kwargs,):@@ -243,6 +247,13 @@ class AutoRoundConfig(QuantizationConfigMixin):self.dataset = datasetself.group_size = group_sizeself.sym = sym+ if "auto" == backend:+ if torch.cuda.is_available():+ backend = "auto_round:exllamav2"+ elif is_hpu_supported():+ backend = "hpu"+ else:+ backend = "cpu"self.backend = backendself.layer_config = layer_configif kwargs is not None:@@ -251,8 +262,6 @@ class AutoRoundConfig(QuantizationConfigMixin):self.quant_method = AutoRoundQuantizationMethod.AutoRoundself.post_init()- def get_loading_attributes(self):- return {}def post_init(self):r"""Safety checker that arguments are correct."""@@ -262,6 +271,12 @@ class AutoRoundConfig(QuantizationConfigMixin):raise ValueError("group_size must be greater than 0 or equal to -1")##TODO add more check+ def get_loading_attributes(self):+ attributes_dict = copy.deepcopy(self.__dict__)+ loading_attributes = ["backend"]+ loading_attibutes_dict = {i: j for i, j in attributes_dict.items() if i in loading_attributes}+ return loading_attibutes_dict+def to_dict(self):config_dict = super().to_dict()config_dict.pop("disable_exllama", None)@@ -296,7 +311,7 @@ class AutoRoundQuantizer(HfQuantizer):def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":if torch_dtype is None:torch_dtype = torch.float16- elif torch_dtype != torch.float16:+ elif torch_dtype != torch.float16 and not is_hpu_supported():logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with AutoRound.")return torch_dtype@@ -311,10 +326,18 @@ class AutoRoundQuantizer(HfQuantizer):layer_names = get_layer_names_in_block(model)quantization_config = model.config.quantization_config+ if hasattr(quantization_config, "backend"): # pragma: no cover+ backend = quantization_config.backend+ if "hpu" in backend and model.dtype != torch.bfloat16:+ logger.info("We suggest you to set `torch_dtype=torch.bfloat16` for better efficiency with AutoRound.")+ model = model.to(torch.bfloat16)+ elif model.dtype != torch.float16:+ logger.info("We suggest you to set `torch_dtype=torch.float16` for better efficiency with AutoRound.")+ model = model.to(torch.float16)bits = quantization_config.bitsgroup_size = quantization_config.group_sizedata_type = quantization_config.data_type if hasattr(quantization_config, "data_type") \- else "int" # pragma: no cover+ else "int" # pragma: no coversym = quantization_config.symextra_config = {}if hasattr(quantization_config, "extra_config"):@@ -334,11 +357,11 @@ class AutoRoundQuantizer(HfQuantizer):layer_configs[layer_name]["group_size"] = extra_config[layer_name].get("group_size", group_size)layer_configs[layer_name]["data_type"] = extra_config[layer_name].get("data_type", data_type)layer_configs[layer_name]["sym"] = extra_config[layer_name].get("sym", sym)- if hasattr(quantization_config, "backend"): # pragma: no cover+ if hasattr(quantization_config, "backend"): # pragma: no coverbackend = quantization_config.backend- elif 'gptq' in quantization_config.quant_method: # pragma: no cover+ elif 'gptq' in quantization_config.quant_method: # pragma: no coverbackend = 'gptq'- else: # pragma: no cover+ else: # pragma: no coverlogger.error("Please specify quantization backend")self._replace_by_quant_layers(model, layer_configs, backend)diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex aefc0d7f4..72ce8a235 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -49,6 +49,7 @@ from .utils import (from .low_cpu_mem.utils import get_layers_before_block+class AutoRound(object):"""This is Signround+ which is an advanced version of Signround. For more information,please refer to Cheng, Wenhua, et al. "Optimize weight rounding via signed gradient descent@@ -151,8 +152,8 @@ class AutoRound(object):self.quantized = Falseself.model_orig_dtype = model.dtypeself.low_cpu_mem_usage = low_cpu_mem_usage- assert not unsupport_meta_device(model), (- "autoround does not support for params on meta device by transformers` interfaces,"+ assert not unsupport_meta_device(model), (+ "autoround does not support for params on meta device by transformers` interfaces,""please do not using device_map='auto' in model loading, ""or follow examples/language-modeling/main.py to enable low_cpu_mem_usage")self.model = model.eval()@@ -222,7 +223,7 @@ class AutoRound(object):assert self.bits > 0, "bits must be positive"assert self.act_bits > 0, "bits must be positive"assert self.group_size == -1 or self.group_size >= 1, "only supports positive group_size or -1(per channel)"- assert self.act_group_size == -1 or self.act_group_size >= 1,\+ assert self.act_group_size == -1 or self.act_group_size >= 1, \"only supports positive group_size or -1(per channel)"assert self.train_bs > 0, "batch size must be positive"assert self.iters > 0, "iters must be positive"@@ -245,7 +246,7 @@ class AutoRound(object):all_blocks = self.quant_block_listelse:all_blocks = get_block_names(self.model)-+if len(all_blocks) == 0:logger.warning("could not find blocks, exit with original model")return self.model, self.layer_config@@ -400,7 +401,6 @@ class AutoRound(object):for key in keys:setattr(m, key, layer_config[n][key])-@torch.no_grad()def get_block_outputs(self, block, input_ids, input_others, bs, device, cache_device):"""Compute the output of a given block of the model for a given input.@@ -473,7 +473,7 @@ class AutoRound(object):embed_layers = get_layers_before_block(self.model)for n, m in embed_layers:m = m.to(self.device)-+for data in self.dataloader:if data is None:continue@@ -490,8 +490,8 @@ class AutoRound(object):data_new[key] = data[key].to(self.device)input_ids = data_new["input_ids"]elif isinstance(data, tuple) or isinstance(data, list):- data_new = data- input_ids = data_new[0]+ data_new = data+ input_ids = data_new[0]else:data_new = {}for key in data.keys():@@ -501,7 +501,7 @@ class AutoRound(object):input_ids = data_new["input_ids"]if input_ids.shape[-1] < self.seqlen:continue-+try:if isinstance(data_new, torch.Tensor):self.model(data_new)@@ -527,7 +527,7 @@ class AutoRound(object):f"Insufficient number of samples collected may affect the quantification. "f"Valid samples size:{total_cnt}, Target sample size:{nsamples}")-+# clean embed weight to save memoryif self.low_cpu_mem_usage:for n, m in embed_layers:@@ -671,8 +671,8 @@ class AutoRound(object):elif "position_ids" in key:if key not in self.inputs[name].keys():self.inputs[name][key] = list(torch.split(kwargs[key].to("cpu"), 1, dim=0)) \- if self.not_share_position_ids_flag \- else to_device(kwargs[key], device=torch.device("cpu"))+ if self.not_share_position_ids_flag \+ else to_device(kwargs[key], device=torch.device("cpu"))elif kwargs[key] is not None and self.not_share_position_ids_flag:self.inputs[name][key].extend(list(torch.split(kwargs[key].to("cpu"), 1, dim=0)))elif key not in self.inputs[name].keys():@@ -1064,7 +1064,7 @@ class AutoRound(object):torch.cuda.empty_cache()- def save_quantized(self, output_dir=None, format="auto_gptq", inplace=True, **kwargs):+ def save_quantized(self, output_dir=None, format="auto_round", inplace=True, **kwargs):"""Save the quantized model to the specified output directory in the specified format.Args:@@ -1529,7 +1529,3 @@ class AutoAdamRound(AutoOPTRound):optimizer,**kwargs,)----diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex efadf0c2b..c95db837c 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -22,6 +22,7 @@ MXFP_FORMAT_CACHE = {"mx_int4": (0, 4, 0, 1.75, 0),"mx_int2": (0, 2, 0, 1.0, 0),"mx_fp8e5m2": (5, 4, 15, 57344.0, 6.103515625e-05),+ "mx_fp8": (4, 5, 8, 448.0, 0.015625),"mx_fp8e4m3": (4, 5, 8, 448.0, 0.015625),"mx_fp6e3m2": (3, 4, 4, 28.0, 0.25),"mx_fp6e2m3": (2, 5, 2, 7.5, 1.0),@@ -73,7 +74,8 @@ def quant_mx(tensor, bits, data_type, v, max_scale, **kwargs):shared_exp[shared_exp > scale_emax] = scale_emax ##changed Nanshared_exp[shared_exp < -scale_emax] = -scale_emaxtensor = tensor / (2 ** shared_exp)- multiply = 2 if data_type == "mx_fp4" else 1 ## 2 is a tricky setting+ is_mx_fp4 = data_type == "mx_fp4" or ("mx_fp" in data_type and bits==4)+ multiply = 2 if is_mx_fp4 else 1 ## 2 is a tricky settingtensor = tensor + v * multiplyif ebits != 0:private_exp = floor_ste(torch.log2(torch.abs(tensor) + (tensor == 0).type(tensor.dtype)))diff --git a/auto_round/utils.py b/auto_round/utils.pyindex a3f2a5e18..ff2d473a4 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -804,7 +804,7 @@ def dynamic_import_inference_linear(backend, bits, group_size, sym):else:from auto_round_extension.hpu.qlinear_hpu_gptq import QuantLinearreturn QuantLinear- if bits == 4 and is_optimum_habana_available(): # pragma: no cover+ if (bits == 4 and is_optimum_habana_available()) or "hpu" in backend: # pragma: no covertry:import habana_frameworks.torch.hpu # noqa: F401 # pylint: disable=E0401except Exception as e:diff --git a/auto_round/version.py b/auto_round/version.pyindex c7c0b0e2a..b2bcf8e76 100644--- a/auto_round/version.py+++ b/auto_round/version.py@@ -14,4 +14,4 @@"""Intel® auto-round: An open-source Python librarysupporting popular model weight only compression based on signround."""-__version__ = "0.3.0.dev"+__version__ = "0.3.1.dev"diff --git a/examples/language-modeling/main.py b/examples/language-modeling/main.pyindex 733447600..0b0548332 100644--- a/examples/language-modeling/main.py+++ b/examples/language-modeling/main.py@@ -81,14 +81,13 @@ if __name__ == '__main__':parser.add_argument("--enable_minmax_tuning", action='store_true',help="enable_minmax_tuning is deprecated")- parser.add_argument("--deployment_device", default='fake', type=str,+ parser.add_argument("--deployment_device", default=None, type=str,help="targeted inference acceleration platform,The options are 'fake', 'cpu', 'gpu' and 'xpu'.""default to 'fake', indicating that it only performs fake quantization and won't be exported to any device.")parser.add_argument("--data_type", default='int',help="data type for tuning, 'int', 'mx_fp' and etc.")-parser.add_argument("--scale_dtype", default='fp16',help="which scale data type to use for quantization, 'fp16', 'fp32' or 'bf16'.")@@ -120,10 +119,10 @@ if __name__ == '__main__':parser.add_argument("--low_cpu_mem_mode", default=0, type=int,help="Choose which low cpu memory mode to use. Can significantly reduce cpu memory footprint but cost more time."- "1 means choose block-wise mode, load the weights of each block from disk when tuning and release the memory of the block after tuning."- "2 means choose layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed."- "others means not use low cpu memory. Default to 0, not use low cpu memory.")-+ "1 means choose block-wise mode, load the weights of each block from disk when tuning and release the memory of the block after tuning."+ "2 means choose layer-wise mode, load the weights of each layer from disk when tuning, minimum memory consumption and also slowest running speed."+ "others means not use low cpu memory. Default to 0, not use low cpu memory.")+parser.add_argument("--low_cpu_mem_tmp_dir", default=None, type=str,help="temp work space to store the temporary files when using low cpu memory mode. Will remove after tuning.")@@ -151,9 +150,12 @@ if __name__ == '__main__':tasks = args.tasksuse_eval_legacy = False+ if args.deployment_device is None:+ args.deployment_device = "auto_round"if "gpu" in args.deployment_device and args.sym is False:- print("warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use --deployment_device='auto_round'")+ print(+ "warning: The auto_gptq kernel has issues with asymmetric quantization. It is recommended to use --deployment_device='auto_round'")if "marlin" in args.deployment_device and args.sym is False:assert False, "marlin backend only supports sym quantization, please set --sym"@@ -187,6 +189,7 @@ if __name__ == '__main__':args.low_cpu_mem_tmp_dir = os.path.join(args.output_dir, "low_cpu_mem_tmp")if args.low_cpu_mem_mode == 2:from auto_round.low_cpu_mem.utils import load_model_with_hooks+model = load_model_with_hooks(model_name,model_cls,@@ -198,6 +201,7 @@ if __name__ == '__main__':)elif args.low_cpu_mem_mode == 1:from auto_round.low_cpu_mem.utils import load_empty_model+low_cpu_mem_usage = Truemodel = load_empty_model(model_name,@@ -206,7 +210,7 @@ if __name__ == '__main__':saved_path=args.low_cpu_mem_tmp_dir,torch_dtype=torch_dtype,trust_remote_code=not args.disable_trust_remote_code- )+ )else:model = model_cls.from_pretrained(model_name, low_cpu_mem_usage=True, torch_dtype=torch_dtype,@@ -305,6 +309,7 @@ if __name__ == '__main__':model_name = args.model_name.rstrip("/")if args.low_cpu_mem_mode == 1 or args.low_cpu_mem_mode == 2:import shutil+shutil.rmtree(args.low_cpu_mem_tmp_dir, ignore_errors=True)model.eval()@@ -336,7 +341,7 @@ if __name__ == '__main__':elif "gptq" in gpu_format:eval_folder = f'{export_dir}-gpu'autoround.save_quantized(eval_folder, format=gpu_format, use_triton=False, inplace=inplace)- elif "auto_awq" in gpu_format: # pragma: no cover+ elif "auto_awq" in gpu_format: # pragma: no covereval_folder = f'{export_dir}-awq'autoround.save_quantized(eval_folder, format=gpu_format, inplace=inplace, model_path=model_name)@@ -419,6 +424,3 @@ if __name__ == '__main__':from lm_eval.utils import make_tableprint(make_table(res))---