Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix set_layer_config bug (#768)

Wenhua Chengbf58cf97f558
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 5e7a7f51c..cc8a526b8 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import contextlib
import copy
import os
import re
@@ -73,7 +72,8 @@ from auto_round.utils import (
is_nv_fp,
is_optimum_habana_available,
is_standard_fp,
- is_static_afp8,
+ is_static_wfp8afp8,
+ is_wfp8afp8,
llm_load_model,
logger,
mv_module_from_gpu,
@@ -215,7 +215,7 @@ class AutoRound(object):
self.vlm = kwargs.pop("vlm") if "vlm" in kwargs else False
if kwargs:
- logger.warning(f"Unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
+ logger.warning(f"unrecognized keys {list(kwargs.keys())} were passed. Please check them.")
if device is not None and "," in str(device):
raise ValueError(
@@ -255,6 +255,12 @@ class AutoRound(object):
if tmp_bits is not None and tmp_bits < 16 and tmp_bits != bits:
logger.warning(f"'data_type' do not match the specified 'bits' setting. Resetting 'bits' to {tmp_bits}.")
self.bits = tmp_bits
+ if tmp_bits is not None and tmp_bits < 16:
+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
+ if data_type.startswith(supported_dtype):
+ if supported_dtype + str(tmp_bits) == self.data_type: # could not replace FP8_e4m3
+ self.data_type = supported_dtype
+ break
self.group_size = group_size
self.sym = sym
self.layer_config = {} if layer_config is None else layer_config
@@ -267,9 +273,9 @@ class AutoRound(object):
self.act_dynamic = act_dynamic
self.act_data_type = act_data_type
if self.act_data_type is None:
- if data_type in SUPPORTED_DTYPES and self.act_bits < 16:
+ if self.data_type in SUPPORTED_DTYPES and self.act_bits < 16:
self.act_data_type = data_type
- logger.info(f"Activation adopts {data_type}")
+ logger.info(f"activation adopts {data_type}")
else:
self.act_data_type = "float"
tmp_act_bits = infer_bits_by_data_type(self.act_data_type)
@@ -279,6 +285,12 @@ class AutoRound(object):
f"`act_data_type` do not"
f" match the specified 'act_bits' setting. Resetting 'act_bits' to {tmp_act_bits}."
)
+ if tmp_act_bits is not None and tmp_act_bits < 16:
+ for supported_dtype in SUPPORTED_DTYPES: # to easily handle dtype mx_fp4 and layer_config={xxx:{bits:8}}
+ if self.act_data_type.startswith(supported_dtype):
+ if supported_dtype + str(tmp_act_bits) == self.act_data_type: # could not replace FP8_e4m3
+ self.act_data_type = supported_dtype
+ break
# Tuning hyperparameters
self.seed = seed
@@ -348,7 +360,7 @@ class AutoRound(object):
self._set_amp_dtype()
self.cache_device = torch.device("cpu") if self.low_gpu_mem_usage else self.device
if self.act_bits <= 8 and self.amp_dtype == torch.float16:
- logger.warning("Force to use bf16 to for quantization tuning when enabling activation quantization")
+ logger.warning("force to use bf16 to for quantization tuning when enabling activation quantization")
self.amp_dtype = torch.bfloat16
if self.model.dtype != torch.bfloat16: # keep the model's buffer dtype unchanged
self.model = self.model.to(torch.bfloat16)
@@ -370,7 +382,7 @@ class AutoRound(object):
torch.set_printoptions(precision=3, sci_mode=True)
if is_optimum_habana_available():
- logger.info("Optimum Habana is available, import htcore explicitly.")
+ logger.info("optimum Habana is available, import htcore explicitly.")
import habana_frameworks.torch.core as htcore # pylint: disable=E0401
import habana_frameworks.torch.hpu as hthpu # pylint: disable=E0401]
@@ -483,7 +495,7 @@ class AutoRound(object):
module = get_module(self.model, name)
if hasattr(module, "tuning_device") and module.tuning_device != device:
logger.warning(
- f"Multiple devices have been set for layer {name}, keeping original device {module.tuning_device}"
+ f"multiple devices have been set for layer {name}, keeping original device {module.tuning_device}"
)
else:
module.tuning_device = device
@@ -504,7 +516,7 @@ class AutoRound(object):
ValueError, TypeError: If any of the configurations are invalid.
"""
if not isinstance(self.model, torch.nn.Module):
- raise TypeError("Model must be an instance of torch.nn.Module")
+ raise TypeError("model must be an instance of torch.nn.Module")
if self.bits <= 0:
raise ValueError("`bits` must be positive")
if self.act_bits <= 0:
@@ -526,28 +538,28 @@ class AutoRound(object):
if self.act_bits <= 8:
logger.warning(
- "Activation quantization is an experimental feature with limited support and a complex API. "
+ "activation quantization is an experimental feature with limited support and a complex API. "
"And please save the quantized model to fake format as real deployment is not supported currently"
)
if is_mx_fp(self.data_type) and self.group_size != 32:
- logger.warning("Dtype mx_fp should only support group_size of 32 in real deployment")
+ logger.warning("dtype mx_fp should only support group_size of 32 in real deployment")
if is_nv_fp(self.data_type) and (self.group_size != 16):
- logger.warning("Dtype nv_fp should only support group_size of 16 in real deployment")
+ logger.warning("dtype nv_fp should only support group_size of 16 in real deployment")
if self.nsamples < self.gradient_accumulate_steps * self.batch_size:
if self.batch_size > self.nsamples:
if self.iters > 0: # GGUF should log this warning, but we don't know the format here
logger.warning(
- f"Reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"
+ f"reset `batch_size` to {self.nsamples} as `nsamples`({self.nsamples})"
f" is smaller than batch_size({self.batch_size})"
)
self.batch_size = self.nsamples
if self.gradient_accumulate_steps > self.nsamples // self.batch_size:
self.gradient_accumulate_steps = self.nsamples // self.batch_size
logger.warning(
- f"Reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"
+ f"reset `gradient_accumulate_steps` to {self.gradient_accumulate_steps}"
f" as nsamples must equal or greater"
f" than gradient_accumulate_steps * batch_size"
)
@@ -657,26 +669,36 @@ class AutoRound(object):
format = formats[index]
if format == "auto_round":
if self.sym and "int" in self.data_type:
- format = format.replace("auto_round", "auto_round:auto_gptq")
- formats[index] = format
- if self.bits == 4 and not self.sym and "int" in self.data_type:
+ format = "auto_round:auto_gptq"
+ elif self.bits == 4 and not self.sym and "int" in self.data_type:
enable_awq = all(
config["bits"] == self.bits or config["bits"] >= 16 for config in self.layer_config.values()
)
if enable_awq:
- formats[index] = format.replace("auto_round", "auto_round:auto_awq")
- if is_nv_fp(self.data_type) or is_mx_fp(self.data_type) or is_standard_fp(self.data_type):
- format = format.replace("auto_round", f"auto_round:{self.data_type}")
- formats[index] = format
+ format = "auto_round:auto_awq"
+ elif is_nv_fp(self.data_type) or is_mx_fp(self.data_type):
+ format = f"auto_round:{self.data_type}"
+ elif is_wfp8afp8(self): # staic wfp8afp8
+ format = "auto_round:fp8"
+ elif self.data_type == "fp" and self.bits == 8 and self.act_bits >= 16: # woq fp8
+ format = "auto_round:fp8"
+ elif self.act_bits < 16:
+ raise ValueError(
+ "AutoRound format does not support exporting "
+ "for the current quantization configuration, "
+ "please change to `fake` format for research purpose"
+ )
+
+ formats[index] = format
elif format == "llmcompressor":
from auto_round.export.export_to_llmcompressor import check_compressed_tensors_supported
if check_compressed_tensors_supported() and (is_nv_fp(self.data_type) or is_mx_fp(self.data_type)):
format = format.replace("llmcompressor", f"llmcompressor:{self.data_type}")
formats[index] = format
- elif not is_standard_fp(self.data_type):
+ elif not is_wfp8afp8(self):
logger.error(
- "Currently, the llmcompressor format only supports MXFP/NVFP/StandardFP8. "
+ "Currently, the llmcompressor format only supports MXFP/NVFP/FP8. "
"Please change format to fake or auto_round etc."
)
@@ -1278,7 +1300,7 @@ class AutoRound(object):
self._quant_rtn_with_imatrix(all_to_quantized_module_names)
elif self.act_bits <= 8 and check_need_act_calibration(
self.act_dynamic, self.act_data_type
- ): ##TODO, mixed datatype has bug
+ ): # TODO, mixed datatype has bug
hook_handles = self._register_act_max_hook(self.model)
try:
self._quantize_via_rtn_blockwise(all_to_quantized_module_names)
@@ -1406,7 +1428,7 @@ class AutoRound(object):
if (
is_nv_fp(self.act_data_type) and any("nv_fp" in format_ for format_ in self.formats)
- ) or is_static_afp8(self):
+ ) or is_static_wfp8afp8(self):
from auto_round.utils import set_amax_for_all_moe_layers
# enable moe experts act_max automatic generation for Linear
@@ -1738,25 +1760,28 @@ class AutoRound(object):
# If the layer is partially configured, fill in missing values
elif n in layer_config.keys():
+ if "data_type" in layer_config[n] and "bits" not in layer_config[n]:
+ tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])
+ if tmp_bits is not None and tmp_bits != self.bits:
+ logger.warning(
+ f"'data_type' do not match the specified 'bits' setting for {n}."
+ f" Resetting 'bits' to {tmp_bits}."
+ )
+ layer_config[n]["bits"] = tmp_bits
+ if "act_data_type" in layer_config[n] and "act_bits" not in layer_config[n]:
+ tmp_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])
+ if tmp_bits is not None and tmp_bits != self.act_bits:
+ logger.warning(
+ f"'act_data_type' do not match the specified 'act_bits' setting for {n}."
+ f" Resetting 'act_bits' to {tmp_bits}."
+ )
+ layer_config[n]["act_bits"] = tmp_bits
+
for key in keys:
if key not in layer_config[n].keys():
layer_config[n][key] = getattr(self, key)
layer_config[n]["fixed_by_user"] = True
- # Check dtype and act_dtype
- tmp_bits = infer_bits_by_data_type(layer_config[n]["data_type"])
- if tmp_bits is not None and tmp_bits != layer_config[n]["bits"]:
- logger.warning(
- f"'data_type' do not match the specified 'bits' setting for {n}."
- f" Resetting 'bits' to {tmp_bits}."
- )
- layer_config[n]["bits"] = tmp_bits
- tmp_act_bits = infer_bits_by_data_type(layer_config[n]["act_data_type"])
- if tmp_act_bits is not None and tmp_act_bits != layer_config[n]["act_bits"]:
- logger.warning(
- f"'act_data_type' do not match the specified 'act_bits' setting for {n}."
- f" Resetting 'act_bits' to {tmp_act_bits}."
- )
- layer_config[n]["act_bits"] = tmp_act_bits
+
# If the layer is not in the config and not part of a quantization block,
# use default configuration and set specific values
else:
diff --git a/auto_round/export/export_to_autoround/export.py b/auto_round/export/export_to_autoround/export.py
index 1640528b6..f56d2fbf1 100644
--- a/auto_round/export/export_to_autoround/export.py
+++ b/auto_round/export/export_to_autoround/export.py
@@ -148,12 +148,12 @@ def pack_layer(layer_name, model, backend):
None: The function modifies the model in place.
"""
if is_nv_fp(backend) or is_mx_fp(backend):
- from auto_round.export.export_to_autoround.export_to_fp import pack_layer
+ from auto_round.export.export_to_autoround.export_to_nvfp_mxfp import pack_layer
return pack_layer(layer_name, model, backend)
- if is_standard_fp(backend):
- from auto_round.export.export_to_autoround.export_to_fp8_woq import pack_layer
+ if backend == "auto_round:fp8":
+ from auto_round.export.export_to_autoround.export_to_fp8 import pack_layer
return pack_layer(layer_name, model, backend)
@@ -265,12 +265,12 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="auto_round:ex
"""
data_type = kwargs.get("data_type", None)
if is_nv_fp(data_type) or is_mx_fp(data_type): ## detect nvfp & mxfp first
- from auto_round.export.export_to_autoround.export_to_fp import save_quantized_as_fp
+ from auto_round.export.export_to_autoround.export_to_nvfp_mxfp import save_quantized_as_fp
return save_quantized_as_fp(output_dir, inplace=inplace, backend="auto_round", **kwargs)
- if is_standard_fp(data_type) and kwargs.get("act_bits", 16) >= 16:
- from auto_round.export.export_to_autoround.export_to_fp8_woq import save_quantized_as_autoround
+ if kwargs.get("data_type", "int") == "fp" and kwargs.get("bits", 16) == 8 and kwargs.get("act_bits", 16) >= 16:
+ from auto_round.export.export_to_autoround.export_to_fp8 import save_quantized_as_autoround
return save_quantized_as_autoround(output_dir, inplace=inplace, backend="auto_round", **kwargs)
diff --git a/auto_round/export/export_to_autoround/export_to_fp8_woq.py b/auto_round/export/export_to_autoround/export_to_fp8.py
similarity index 100%
rename from auto_round/export/export_to_autoround/export_to_fp8_woq.py
rename to auto_round/export/export_to_autoround/export_to_fp8.py
diff --git a/auto_round/export/export_to_autoround/export_to_fp.py b/auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
similarity index 100%
rename from auto_round/export/export_to_autoround/export_to_fp.py
rename to auto_round/export/export_to_autoround/export_to_nvfp_mxfp.py
diff --git a/auto_round/utils.py b/auto_round/utils.py
index a87aea892..c3606aafb 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -2502,13 +2502,13 @@ def set_amax_for_all_moe_layers(model: torch.nn.Module, layer_name=None, attr_na
class BackendDataType(str, Enum):
- STANDARD_FP8 = "fp8"
+ STANDARD_FP = "fp"
MX_FP = "mx_fp"
NV_FP = "nv_fp"
def is_standard_fp(backend):
- return BackendDataType.STANDARD_FP8 in backend and not is_mx_fp(backend) and not is_nv_fp(backend)
+ return BackendDataType.STANDARD_FP in backend and not is_mx_fp(backend) and not is_nv_fp(backend)
def is_mx_fp(backend):
@@ -2519,5 +2519,18 @@ def is_nv_fp(backend):
return BackendDataType.NV_FP in backend
-def is_static_afp8(ar):
- return not ar.act_dynamic and "fp8" in ar.act_data_type
+def is_wfp8afp8(ar):
+ if ("fp8" in ar.act_data_type or ("fp" in ar.act_data_type and ar.act_bits == 8)) and (
+ "fp8" in ar.data_type or ("fp" in ar.data_type and ar.bits == 8)
+ ):
+ return True
+ else:
+ return False
+
+
+def is_static_wfp8afp8(ar):
+ if ar.act_dynamic:
+ return False
+ if is_wfp8afp8(ar):
+ return True
+ return False
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index 86de61341..1f2b02d9b 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -604,6 +604,18 @@ class TestAutoRound(unittest.TestCase):
self.assertEqual(dequant_weight.shape[0], 32)
self.assertEqual(dequant_weight.shape.numel(), 32 * 5760 * 1440)
+ def test_mixed_bit_setting(self):
+ model_name = "facebook/opt-125m"
+ layer_config = {"model.decoder.layers.7.fc1": {"bits": 8, "act_bits": 8}}
+ ar = AutoRound(model_name, data_type="mx_fp4", act_bits=4, iters=0, layer_config=layer_config)
+ ar.quantize()
+ layer_config = ar.layer_config
+ if (
+ layer_config["model.decoder.layers.7.fc1"]["bits"] != 8
+ or layer_config["model.decoder.layers.7.fc1"]["act_bits"] != 8
+ ):
+ raise ValueError("mixed bits is not correct")
+
if __name__ == "__main__":
unittest.main()