PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix set_layer_config bug (#768)
Wenhua Chengbf58cf97f558
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex 5e7a7f51c..cc8a526b8 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -12,7 +12,6 @@# See the License for the specific language governing permissions and# limitations under the License.-import contextlibimport copyimport osimport re@@ -73,7 +72,8 @@ from auto_round.utils import (is_nv_fp,is_optimum_habana_available,is_standard_fp,- is_static_afp8,+ is_static_wfp8afp8,+ is_wfp8afp8,llm_load_model,logger,mv_module_from_gpu,@@ -215,7 +215,7 @@ class AutoRound(object):self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else Falseif kwargs:- logger.warning(f"Unrecognized keys {list(kwargs.keys())} were passed. Please check them.")+ logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")if device is not None and "," in str(device):raise ValueError(@@ -255,6 +255,12 @@ class AutoRound(object):if tmp_bits is not None and tmp_bits < 16 and tmp_bits != bits:logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")self.bits = tmp_bits+ if tmp_bits is not None and tmp_bits < 16:+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}+ if data_type.startswith(supported_dtype):+ if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3+ self.data_type = supported_dtype+ breakself.group_size = group_sizeself.sym = symself.layer_config = {} if layer_config is None else layer_config@@ -267,9 +273,9 @@ class AutoRound(object):self.act_dynamic = act_dynamicself.act_data_type = act_data_typeif self.act_data_type is None:- if data_type in SUPPORTED_DTYPES and self.act_bits < 16:+ if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:self.act_data_type = data_type- logger.info(f"Activation adopts {data_type}")+ logger.info(f"activation adopts {data_type}")else:self.act_data_type = "float"tmp_act_bits = infer_bits_by_data_type(self.act_data_type)@@ -279,6 +285,12 @@ class AutoRound(object):f"`act_data_type` do not"f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}.")+ if tmp_act_bits is not None and tmp_act_bits < 16:+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}+ if self.act_data_type.startswith(supported_dtype):+ if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3+ self.act_data_type = supported_dtype+ break# Tuning hyperparametersself.seed = seed@@ -348,7 +360,7 @@ class AutoRound(object):self._set_amp_dtype()self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.deviceif self.act_bits <= 8 and self.amp_dtype == torch.float16:- logger.warning("Force to use bf16 to for quantization tuning when enabling activation quantization")+ logger.warning("force to use bf16 to for quantization tuning when enabling activation quantization")self.amp_dtype = torch.bfloat16if self.model.dtype != torch.bfloat16: # keep the model's buffer dtype unchangedself.model = self.model.to(torch.bfloat16)@@ -370,7 +382,7 @@ class AutoRound(object):torch.set_printoptions(precision=3, sci_mode=True)if is_optimum_habana_available():- logger.info("Optimum Habana is available, import htcore explicitly.")+ logger.info("optimum Habana is available, import htcore explicitly.")import habana_frameworks.torch.core as htcore # pylint: disable=E0401import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]@@ -483,7 +495,7 @@ class AutoRound(object):module = get_module(self.model, name)if hasattr(module, "tuning_device") and module.tuning_device != device:logger.warning(- f"Multiple devices have been set for layer {name}, keeping original device {module.tuning_device}"+ f"multiple devices have been set for layer {name}, keeping original device {module.tuning_device}")else:module.tuning_device = device@@ -504,7 +516,7 @@ class AutoRound(object):ValueError, TypeError: If any of the configurations are invalid."""if not isinstance(self.model, torch.nn.Module):- raise TypeError("Model must be an instance of torch.nn.Module")+ raise TypeError("model must be an instance of torch.nn.Module")if self.bits <= 0:raise ValueError("`bits` must be positive")if self.act_bits <= 0:@@ -526,28 +538,28 @@ class AutoRound(object):if self.act_bits <= 8:logger.warning(- "Activation quantization is an experimental feature with limited support and a complex API. "+ "activation quantization is an experimental feature with limited support and a complex API. ""And please save the quantized model to fake format as real deployment is not supported currently")if is_mx_fp(self.data_type) and self.group_size != 32:- logger.warning("Dtype mx_fp should only support group_size of 32 in real deployment")+ logger.warning("dtype mx_fp should only support group_size of 32 in real deployment")if is_nv_fp(self.data_type) and (self.group_size != 16):- logger.warning("Dtype nv_fp should only support group_size of 16 in real deployment")+ logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")if self.nsamples < self.gradient_accumulate_steps * self.batch_size:if self.batch_size > self.nsamples:if self.iters > 0: # GGUF should log this warning, but we don't know the format herelogger.warning(- f"Reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"+ f"reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"f" is smaller than batch_size({self.batch_size})")self.batch_size = self.nsamplesif self.gradient_accumulate_steps > self.nsamples // self.batch_size:self.gradient_accumulate_steps = self.nsamples // self.batch_sizelogger.warning(- f"Reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"+ f"reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"f" as nsamples must equal or greater"f" than gradient_accumulate_steps * batch_size")@@ -657,26 +669,36 @@ class AutoRound(object):format = formats[index]if format == "auto_round":if self.sym and "int" in self.data_type:- format = format.replace("auto_round", "auto_round:auto_gptq")- formats[index] = format- if self.bits == 4 and not self.sym and "int" in self.data_type:+ format = "auto_round:auto_gptq"+ elif self.bits == 4 and not self.sym and "int" in self.data_type:enable_awq = all(config["bits"] == self.bits or config["bits"] >= 16 for config in self.layer_config.values())if enable_awq:- formats[index] = format.replace("auto_round", "auto_round:auto_awq")- if is_nv_fp(self.data_type) or is_mx_fp(self.data_type) or is_standard_fp(self.data_type):- format = format.replace("auto_round", f"auto_round:{self.data_type}")- formats[index] = format+ format = "auto_round:auto_awq"+ elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):+ format = f"auto_round:{self.data_type}"+ elif is_wfp8afp8(self): # staic wfp8afp8+ format = "auto_round:fp8"+ elif self.data_type == "fp" and self.bits == 8 and self.act_bits >= 16: # woq fp8+ format = "auto_round:fp8"+ elif self.act_bits < 16:+ raise ValueError(+ "AutoRound format does not support exporting "+ "for the current quantization configuration, "+ "please change to `fake` format for research purpose"+ )++ formats[index] = formatelif format == "llmcompressor":from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supportedif check_compressed_tensors_supported() and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):format = format.replace("llmcompressor", f"llmcompressor:{self.data_type}")formats[index] = format- elif not is_standard_fp(self.data_type):+ elif not is_wfp8afp8(self):logger.error(- "Currently, the llmcompressor format only supports MXFP/NVFP/StandardFP8. "+ "Currently, the llmcompressor format only supports MXFP/NVFP/FP8. ""Please change format to fake or auto_round etc.")@@ -1278,7 +1300,7 @@ class AutoRound(object):self._quant_rtn_with_imatrix(all_to_quantized_module_names)elif self.act_bits <= 8 and check_need_act_calibration(self.act_dynamic, self.act_data_type- ): ##TODO, mixed datatype has bug+ ): # TODO, mixed datatype has bughook_handles = self._register_act_max_hook(self.model)try:self._quantize_via_rtn_blockwise(all_to_quantized_module_names)@@ -1406,7 +1428,7 @@ class AutoRound(object):if (is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)- ) or is_static_afp8(self):+ ) or is_static_wfp8afp8(self):from auto_round.utils import set_amax_for_all_moe_layers# enable moe experts act_max automatic generation for Linear@@ -1738,25 +1760,28 @@ class AutoRound(object):# If the layer is partially configured, fill in missing valueselif n in layer_config.keys():+ if "data_type" in layer_config[n] and "bits" not in layer_config[n]:+ tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])+ if tmp_bits is not None and tmp_bits != self.bits:+ logger.warning(+ f"'data_type' do not match the specified 'bits' setting for {n}."+ f" Resetting 'bits' to {tmp_bits}."+ )+ layer_config[n]["bits"] = tmp_bits+ if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]:+ tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])+ if tmp_bits is not None and tmp_bits != self.act_bits:+ logger.warning(+ f"'act_data_type' do not match the specified 'act_bits' setting for {n}."+ f" Resetting 'act_bits' to {tmp_bits}."+ )+ layer_config[n]["act_bits"] = tmp_bits+for key in keys:if key not in layer_config[n].keys():layer_config[n][key] = getattr(self, key)layer_config[n]["fixed_by_user"] = True- # Check dtype and act_dtype- tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])- if tmp_bits is not None and tmp_bits != layer_config[n]["bits"]:- logger.warning(- f"'data_type' do not match the specified 'bits' setting for {n}."- f" Resetting 'bits' to {tmp_bits}."- )- layer_config[n]["bits"] = tmp_bits- tmp_act_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])- if tmp_act_bits is not None and tmp_act_bits != layer_config[n]["act_bits"]:- logger.warning(- f"'act_data_type' do not match the specified 'act_bits' setting for {n}."- f" Resetting 'act_bits' to {tmp_act_bits}."- )- layer_config[n]["act_bits"] = tmp_act_bits+# If the layer is not in the config and not part of a quantization block,# use default configuration and set specific valueselse:diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.pyindex 1640528b6..f56d2fbf1 100644--- a/auto_round/export/export_to_autoround/export.py+++ b/auto_round/export/export_to_autoround/export.py@@ -148,12 +148,12 @@ def pack_layer(layer_name, model, backend):None: The function modifies the model in place."""if is_nv_fp(backend) or is_mx_fp(backend):- from auto_round.export.export_to_autoround.export_to_fp import pack_layer+ from auto_round.export.export_to_autoround.export_to_nvfp_mxfp import pack_layerreturn pack_layer(layer_name, model, backend)- if is_standard_fp(backend):- from auto_round.export.export_to_autoround.export_to_fp8_woq import pack_layer+ if backend == "auto_round:fp8":+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layerreturn pack_layer(layer_name, model, backend)@@ -265,12 +265,12 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex"""data_type = kwargs.get("data_type", None)if is_nv_fp(data_type) or is_mx_fp(data_type): ## detect nvfp & mxfp first- from auto_round.export.export_to_autoround.export_to_fp import save_quantized_as_fp+ from auto_round.export.export_to_autoround.export_to_nvfp_mxfp import save_quantized_as_fpreturn save_quantized_as_fp(output_dir, inplace=inplace, backend="auto_round", **kwargs)- if is_standard_fp(data_type) and kwargs.get("act_bits", 16) >= 16:- from auto_round.export.export_to_autoround.export_to_fp8_woq import save_quantized_as_autoround+ if kwargs.get("data_type", "int") == "fp" and kwargs.get("bits", 16) == 8 and kwargs.get("act_bits", 16) >= 16:+ from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoroundreturn save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8.pysimilarity index 100%rename from auto_round/export/export_to_autoround/export_to_fp8_woq.pyrename to auto_round/export/export_to_autoround/export_to_fp8.pydiff --git a/auto_round/export/export_to_autoround/export_to_fp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pysimilarity index 100%rename from auto_round/export/export_to_autoround/export_to_fp.pyrename to auto_round/export/export_to_autoround/export_to_nvfp_mxfp.pydiff --git a/auto_round/utils.py b/auto_round/utils.pyindex a87aea892..c3606aafb 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -2502,13 +2502,13 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_naclass BackendDataType(str, Enum):- STANDARD_FP8 = "fp8"+ STANDARD_FP = "fp"MX_FP = "mx_fp"NV_FP = "nv_fp"def is_standard_fp(backend):- return BackendDataType.STANDARD_FP8 in backend and not is_mx_fp(backend) and not is_nv_fp(backend)+ return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)def is_mx_fp(backend):@@ -2519,5 +2519,18 @@ def is_nv_fp(backend):return BackendDataType.NV_FP in backend-def is_static_afp8(ar):- return not ar.act_dynamic and "fp8" in ar.act_data_type+def is_wfp8afp8(ar):+ if ("fp8" in ar.act_data_type or ("fp" in ar.act_data_type and ar.act_bits == 8)) and (+ "fp8" in ar.data_type or ("fp" in ar.data_type and ar.bits == 8)+ ):+ return True+ else:+ return False+++def is_static_wfp8afp8(ar):+ if ar.act_dynamic:+ return False+ if is_wfp8afp8(ar):+ return True+ return Falsediff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex 86de61341..1f2b02d9b 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -604,6 +604,18 @@ class TestAutoRound(unittest.TestCase):self.assertEqual(dequant_weight.shape[0], 32)self.assertEqual(dequant_weight.shape.numel(), 32 * 5760 * 1440)+ def test_mixed_bit_setting(self):+ model_name = "facebook/opt-125m"+ layer_config = {"model.decoder.layers.7.fc1": {"bits": 8, "act_bits": 8}}+ ar = AutoRound(model_name, data_type="mx_fp4", act_bits=4, iters=0, layer_config=layer_config)+ ar.quantize()+ layer_config = ar.layer_config+ if (+ layer_config["model.decoder.layers.7.fc1"]["bits"] != 8+ or layer_config["model.decoder.layers.7.fc1"]["act_bits"] != 8+ ):+ raise ValueError("mixed bits is not correct")+if __name__ == "__main__":unittest.main()