Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

mllm supports loading fp8 model and fix bug of loading fp8 model (#750)

Heng Guoc1e719b92561
diff --git a/auto_round/__main__.py b/auto_round/__main__.py
index e79383efd..3cd8ce2d9 100644
--- a/auto_round/__main__.py
+++ b/auto_round/__main__.py
@@ -54,11 +54,8 @@ def run():
def run_mllm():
- from auto_round.script.llm import setup_parser, tune
-
- args = setup_parser()
- args.mllm = True
- tune(args)
+ sys.argv.append("--mllm")
+ run()
def run_best():
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 20d01241d..96063ca6a 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1140,7 +1140,8 @@ class AutoRound(object):
"""
m = get_module(self.model, name)
- if m.__class__.__name__ == "FP8Linear":
+ # if m.__class__.__name__ == "FP8Linear":
+ if hasattr(m, "is_fp8_linear"):
m = convert_fp8_layer_to_linear(m, self.amp_dtype)
set_module(self.model, name, m)
@@ -1532,7 +1533,7 @@ class AutoRound(object):
if hasattr(self.model, "is_fp8"):
for n, m in self.model.named_modules():
- if m.__class__.__name__ == "FP8Linear":
+ if hasattr(m, "is_fp8_linear"):
new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to("cpu")
set_module(self.model, n, new_layer)
@@ -1580,7 +1581,7 @@ class AutoRound(object):
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
layer = get_module(self.model, layer_name)
- if layer.__class__.__name__ == "FP8Linear":
+ if hasattr(layer, "is_fp8"):
new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype).to(self.device)
set_module(self.model, layer_name, new_layer)
layer = new_layer
@@ -2344,7 +2345,7 @@ class AutoRound(object):
"""
if hasattr(self.model, "is_fp8"):
for n, m in block.named_modules():
- if m.__class__.__name__ == "FP8Linear":
+ if hasattr(m, "is_fp8_linear"):
new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to(device)
set_module(block, n, new_layer)
diff --git a/auto_round/utils.py b/auto_round/utils.py
index b9462a500..3ab23e08b 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1343,6 +1343,20 @@ def set_fake_cuda_device_capability(func=None):
return orig_func
+def check_and_mark_fp8_model(model: torch.nn.Module) -> bool:
+ if hasattr(model, "is_fp8"):
+ return model.is_fp8
+ for n, m in model.named_modules():
+ if isinstance(m, torch.nn.Linear) and str(m.weight.dtype).startswith("torch.float8"):
+ m.is_fp8_linear = True
+ if not hasattr(model, "is_fp8"):
+ logger.warning("the support for fp8 model as input is experimental, please use with caution.")
+ model.is_fp8 = True
+ if hasattr(model, "is_fp8"):
+ return True
+ return False
+
+
def llm_load_model(
pretrained_model_name_or_path,
trust_remote_code=True,
@@ -1403,7 +1417,6 @@ def llm_load_model(
)
else:
try:
-
model = model_cls.from_pretrained(
pretrained_model_name_or_path,
torch_dtype=torch_dtype,
@@ -1435,6 +1448,7 @@ def llm_load_model(
)
model = model.eval()
+ check_and_mark_fp8_model(model)
model = _to_model_dtype(model, model_dtype)
return model, tokenizer, low_cpu_mem_usage
@@ -1516,12 +1530,25 @@ def mllm_load_model(
cls = getattr(transformers, architectures)
else:
cls = AutoModelForCausalLM
- model = cls.from_pretrained(
- pretrained_model_name_or_path,
- trust_remote_code=trust_remote_code,
- torch_dtype=torch_dtype,
- device_map="auto" if use_auto_mapping else None,
- )
+ try:
+ model = cls.from_pretrained(
+ pretrained_model_name_or_path,
+ trust_remote_code=trust_remote_code,
+ torch_dtype=torch_dtype,
+ device_map="auto" if use_auto_mapping else None,
+ )
+ except ValueError as e:
+ if "FP8 quantized" in str(e):
+ orig_func = set_fake_cuda_device_capability()
+ model = cls.from_pretrained(
+ pretrained_model_name_or_path,
+ trust_remote_code=trust_remote_code,
+ torch_dtype=torch_dtype,
+ device_map="auto" if use_auto_mapping else None,
+ )
+ torch.cuda.get_device_capability = orig_func
+ model.is_fp8 = True ##tricky setting
+ logger.warning("the support for fp8 model as input is experimental, please use with caution.")
if "Mistral-Small-3.2" in pretrained_model_name_or_path:
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer # pylint: disable=E0401
@@ -1547,6 +1574,7 @@ def mllm_load_model(
pass
model = model.eval()
+ check_and_mark_fp8_model(model)
model = _to_model_dtype(model, model_dtype)
return model, processor, tokenizer, image_processor
@@ -2208,12 +2236,18 @@ def dequant_block_fp8_weight(weight: torch.Tensor, weight_scale: torch.Tensor, b
def convert_fp8_layer_to_linear(layer, dtype=torch.bfloat16):
""" """
new_layer = torch.nn.Linear(layer.in_features, layer.out_features, bias=layer.bias is not None, dtype=dtype)
- if layer.bias:
+ if layer.bias is not None:
new_layer.bias.data.copy_(layer.bias.data.to(dtype=dtype))
+
keys = get_quant_keys() + ["tmp_name"]
for key in keys:
setattr(new_layer, key, getattr(layer, key, None))
- dq_weight = dequant_block_fp8_weight(layer.weight, layer.weight_scale_inv, layer.block_size)
+
+ if layer.__class__.__name__ == "CompressedLinear":
+ dq_weight = layer.compressor.decompress_module(layer)
+ else:
+ weight_scale = layer.weight_scale if hasattr(layer, "weight_scale") else layer.weight_scale_inv
+ dq_weight = dequant_block_fp8_weight(layer.weight, weight_scale, layer.block_size)
new_layer.weight.data.copy_(dq_weight.to(dtype=dtype))
return new_layer