Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

gguf format supports for fp8 model (#778)

Heng Guo572aad1db5a2
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index 4d693ece2..f507d4b18 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -40,6 +40,8 @@ from auto_round.utils import (
TORCH_VERSION_AT_LEAST_2_6,
CpuInfo,
_gguf_args_check,
+ _is_fp8_linear,
+ _is_fp8_model,
block_forward,
check_and_mark_fp8_model,
check_is_cpu,
@@ -957,7 +959,7 @@ class AutoRound(object):
# Load dataset
from auto_round.calib_dataset import get_dataloader
- if hasattr(self.model, "is_fp8"):
+ if _is_fp8_model(self.model):
convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
if isinstance(self.dataset, str):
@@ -1183,7 +1185,7 @@ class AutoRound(object):
m = get_module(self.model, name)
# if m.__class__.__name__ == "FP8Linear":
- if hasattr(m, "is_fp8_linear"):
+ if _is_fp8_linear(m):
m = convert_fp8_layer_to_linear(m, self.amp_dtype)
set_module(self.model, name, m)
@@ -1333,7 +1335,7 @@ class AutoRound(object):
cnt = 1
cnt += 1
# Convert remaining fp8
- if hasattr(self.model, "is_fp8"):
+ if _is_fp8_model(self.model):
convert_fp8_model_to_16b_model(self.model, self.amp_dtype)
self.quantized = True
return self.model, self.layer_config
@@ -1401,7 +1403,7 @@ class AutoRound(object):
pbar.set_description(f"Quantizing {block_name}")
block = get_module(self.model, block_name)
block = block.to(self.device)
- if hasattr(self.model, "is_fp8"):
+ if _is_fp8_model(self.model):
convert_fp8_model_to_16b_model(block, dtype=self.amp_dtype)
# Dispatch model if needed
if self.device_map is not None:
@@ -1586,9 +1588,9 @@ class AutoRound(object):
self._quantize_layers(layer_names, all_inputs) ##TODO pack layer immediately
- if hasattr(self.model, "is_fp8"):
+ if _is_fp8_model(self.model):
for n, m in self.model.named_modules():
- if hasattr(m, "is_fp8_linear"):
+ if _is_fp8_linear(m):
new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to("cpu")
set_module(self.model, n, new_layer)
@@ -1636,7 +1638,7 @@ class AutoRound(object):
from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
layer = get_module(self.model, layer_name)
- if hasattr(layer, "is_fp8"):
+ if _is_fp8_model(self.model):
new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype).to(self.device)
set_module(self.model, layer_name, new_layer)
layer = new_layer
@@ -1997,7 +1999,7 @@ class AutoRound(object):
Raises:
Exception: If caching on GPU fails, switches to CPU and caches there.
"""
- if hasattr(self.model, "is_fp8"):
+ if _is_fp8_model(self.model):
layer_names = []
if layer_names is None:
layer_names = []
@@ -2452,9 +2454,9 @@ class AutoRound(object):
Returns:
Tuple: (q_outputs, output) if self.enable_quanted_input is True, else (None, output)
"""
- if hasattr(self.model, "is_fp8"):
+ if _is_fp8_model(self.model):
for n, m in block.named_modules():
- if hasattr(m, "is_fp8_linear"):
+ if _is_fp8_linear(m):
new_layer = convert_fp8_layer_to_linear(m, self.amp_dtype).to(device)
set_module(block, n, new_layer)
diff --git a/auto_round/export/export_to_gguf/convert.py b/auto_round/export/export_to_gguf/convert.py
index 0072424fb..6cefa11a5 100644
--- a/auto_round/export/export_to_gguf/convert.py
+++ b/auto_round/export/export_to_gguf/convert.py
@@ -50,7 +50,7 @@ from transformers import AutoConfig
from auto_round.export.export_to_gguf.config import ModelType
from auto_round.export.export_to_gguf.packing import ggml_quant
-from auto_round.utils import LazyImport, clean_module_parameter, get_module, logger
+from auto_round.utils import LazyImport, _is_fp8_model, clean_module_parameter, get_module, logger
gguf = LazyImport("gguf")
@@ -144,6 +144,13 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
break
yield name, tensor
+ def is_extra_tensor(tensor_name):
+ if _is_fp8_model(cls.model) and "scale" in tensor_name.split(".")[-1]:
+ return False
+ if tensor_name not in cls.model.tensor_name_list:
+ return True
+ return False
+
extra_tensor = {}
if hasattr(cls.model, "name_or_path"):
from safetensors import safe_open
@@ -159,12 +166,12 @@ def get_tensors(cls) -> Iterator[tuple[str, Tensor]]:
with open(os.path.join(dir_path, INDEX_FILE)) as f:
tensor_index = json.load(f)
for tensor_name in tensor_index["weight_map"]:
- if tensor_name not in cls.model.tensor_name_list:
+ if is_extra_tensor(tensor_name):
extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)
else:
f = safe_open(os.path.join(dir_path, "model.safetensors"), framework="pt")
for tensor_name in f.keys():
- if tensor_name not in cls.model.tensor_name_list:
+ if is_extra_tensor(tensor_name):
extra_tensor[tensor_name] = get_tensor_from_file(dir_path, tensor_name)
for name, tensor in extra_tensor.items():
diff --git a/auto_round/utils.py b/auto_round/utils.py
index c3606aafb..2f93dddcc 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -1366,11 +1366,31 @@ def set_fake_cuda_device_capability(func=None):
return orig_func
-def check_and_mark_fp8_model(model: torch.nn.Module) -> bool:
- if hasattr(model, "is_fp8"):
+def _is_fp8_model(model: torch.nn.Module) -> bool:
+ if not hasattr(model, "is_fp8"):
+ return False
+ else:
return model.is_fp8
+
+
+def _is_fp8_linear(module: torch.nn.Module) -> bool:
+ if hasattr(module, "is_fp8_linear"):
+ return module.is_fp8_linear
+ if not (isinstance(module, torch.nn.Linear) or module.__class__.__name__ == "FP8Linear"):
+ return False
+ if module.weight is None:
+ return False
+ if str(module.weight.dtype).startswith("torch.float8"):
+ return True
+ else:
+ return False
+
+
+def check_and_mark_fp8_model(model: torch.nn.Module) -> bool:
+ if _is_fp8_model(model):
+ return True
for n, m in model.named_modules():
- if isinstance(m, torch.nn.Linear) and str(m.weight.dtype).startswith("torch.float8"):
+ if _is_fp8_linear(m):
m.is_fp8_linear = True
if not hasattr(model, "is_fp8"):
logger.warning("the support for fp8 model as input is experimental, please use with caution.")
@@ -1456,7 +1476,6 @@ def llm_load_model(
device_map="auto" if use_auto_mapping else None,
)
torch.cuda.get_device_capability = orig_func
- model.is_fp8 = True ##tricky setting
logger.warning("the support for fp8 model as input is experimental, please use with caution.")
except OSError as e:
@@ -1569,7 +1588,6 @@ def mllm_load_model(
device_map="auto" if use_auto_mapping else None,
)
torch.cuda.get_device_capability = orig_func
- model.is_fp8 = True ##tricky setting
logger.warning("the support for fp8 model as input is experimental, please use with caution.")
if "Mistral-Small-3.2" in pretrained_model_name_or_path:
diff --git a/test/test_cpu/test_low_cpu_mem.py b/test/test_cpu/test_low_cpu_mem.py
index 85869dde4..13eb34767 100644
--- a/test/test_cpu/test_low_cpu_mem.py
+++ b/test/test_cpu/test_low_cpu_mem.py
@@ -46,8 +46,9 @@ class TestLowCPUMem(unittest.TestCase):
def test_default(self):
self.assertTrue(self.model.device.type, "meta")
- layers = get_layers_before_block(self.model)
- self.assertEqual(layers[0][0], "model.decoder.embed_tokens")
+ # TODO: change this func
+ # layers = get_layers_before_block(self.model)
+ # self.assertEqual(layers[0][0], "model.decoder.embed_tokens")
# test get_weight bias
self.assertTrue(
@@ -68,9 +69,15 @@ class TestLowCPUMem(unittest.TestCase):
input = self.tokenizer(text)
for key in input:
input[key] = torch.tensor(input[key])
- ori_output = self.ori_model(**input)
- output = self.model(**input)
- self.assertTrue(torch.equal(ori_output[0], output[0]))
+ ori_output = self.ori_model.generate(**input, max_new_tokens=5, do_sample=False)
+ ori_result = self.tokenizer.decode(ori_output[0])
+ print(ori_result)
+ self.model.to("cpu")
+ output = self.model.generate(**input, max_new_tokens=5, do_sample=False)
+ result = self.tokenizer.decode(output[0])
+ print(result)
+ self.assertEqual(ori_result, result)
+ self.model.to("meta")
# test save and load
layer_wise_save(self.model, self.saved_path)
diff --git a/test/test_cuda/test_fp8_input.py b/test/test_cuda/test_fp8_input.py
index 00af4ee14..43e745c4e 100644
--- a/test/test_cuda/test_fp8_input.py
+++ b/test/test_cuda/test_fp8_input.py
@@ -3,14 +3,13 @@ import shutil
import sys
import unittest
-from auto_round.eval.evaluation import simple_evaluate
-
sys.path.insert(0, "../..")
import torch
import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer
from auto_round import AutoRound
+from auto_round.eval.evaluation import simple_evaluate
class TestAutoRound(unittest.TestCase):
@@ -96,6 +95,31 @@ class TestAutoRound(unittest.TestCase):
shutil.rmtree(self.save_dir, ignore_errors=True)
+ def test_fp8_model_gguf(self):
+ from llama_cpp import Llama
+
+ model_name = "Qwen/Qwen3-0.6B-FP8"
+
+ ar = AutoRound(model=model_name, iters=0)
+ ar.quantize_and_save(output_dir=self.save_dir, format="gguf:q4_0")
+ for file in os.listdir(self.save_dir):
+ if file.endswith(".gguf"):
+ gguf_file = file
+ llm = Llama(f"saved/{gguf_file}", n_gpu_layers=-1)
+ output = llm("There is a girl who likes adventure,", max_tokens=32)
+ print(output)
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
+ ar = AutoRound(model=model_name, iters=1)
+ ar.quantize_and_save(output_dir=self.save_dir, format="gguf:q3_k_s")
+ for file in os.listdir(self.save_dir):
+ if file.endswith(".gguf"):
+ gguf_file = file
+ llm = Llama(f"saved/{gguf_file}", n_gpu_layers=-1)
+ output = llm("There is a girl who likes adventure,", max_tokens=32)
+ print(output)
+ shutil.rmtree(self.save_dir, ignore_errors=True)
+
if __name__ == "__main__":
unittest.main()