Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix bug of loading fp8 model (#747)

Heng Guo0029322760d0
diff --git a/auto_round/utils.py b/auto_round/utils.py
index 2b8ef540e..4f47a3aa4 100644
--- a/auto_round/utils.py
+++ b/auto_round/utils.py
@@ -24,6 +24,7 @@ import sys
from collections import UserDict
from enum import Enum
from functools import lru_cache
+from typing import Any, Tuple, Union
import cpuinfo
import psutil
@@ -2133,12 +2134,55 @@ def check_need_act_calibration(is_act_dynamic, act_data_type=None):
return False
-def dequant_block_fp8_weight(weight, weight_scale, block_size):
+def pad_weight(weight: torch.Tensor, block_size: list) -> Tuple[torch.Tensor, int, int]:
+ """Pads a matrix to make its dimensions multiples of block_size."""
+ M, N = weight.shape[-2:]
+ block_size_m, block_size_n = block_size
+ pad_M = (block_size_m - M % block_size_m) % block_size_m
+ pad_N = (block_size_n - N % block_size_n) % block_size_n
+
+ if pad_M == 0 and pad_N == 0:
+ return weight, M, N # No padding needed
+ padded_weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)
+ return padded_weight, M, N # Return original dimensions for unpadding
+
+
+def unpad_weight(weight: torch.Tensor, original_M: int, original_N: int, keep_first_dim: bool = False) -> torch.Tensor:
+ """Removes padding from the matrix to restore its original shape."""
+ if (weight.shape[-2] == original_M) and (weight.shape[-1] == original_N):
+ return weight
+ if keep_first_dim:
+ return weight[:, :original_M, :original_N]
+ else:
+ return weight[:original_M, :original_N]
+
+
+def pad_block_fp8_weight_naive(
+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list
+) -> Tuple[torch.Tensor, int, int]:
+
+ assert len(block_size) == 2
+
+ block_size_m, block_size_n = block_size
+ weight_scale_m, weight_scale_n = weight_scale.shape[-2:]
+
+ weight, orig_M, orig_N = pad_weight(weight, block_size)
+ M, N = weight.shape[-2:]
+
+ assert weight_scale_m == M // block_size_m
+ assert weight_scale_n == N // block_size_n
+
+ return weight, orig_M, orig_N
+
+
+def dequant_block_fp8_weight(weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list) -> torch.Tensor:
dtype = torch.bfloat16
if weight_scale is None:
return weight
assert len(block_size) == 2
+ weight, orig_M, orig_N = pad_block_fp8_weight_naive(weight, weight_scale, block_size)
+
weight_shape_len = len(weight.shape)
block_size_m, block_size_n = block_size
@@ -2150,15 +2194,19 @@ def dequant_block_fp8_weight(weight, weight_scale, block_size):
weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)
dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)
+ keep_first_dim = False
elif weight_shape_len == 3:
fd, weight_scale_m, weight_scale_n = weight_scale.shape
weight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)
weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)
dequant_weight = weight.to(dtype) * weight_scale.to(dtype)
dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)
+ keep_first_dim = True
else:
raise ValueError("Only support original weight shape is either 2 or 3")
+ dequant_weight = unpad_weight(dequant_weight, orig_M, orig_N, keep_first_dim=keep_first_dim)
+
return dequant_weight
diff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.py
index afbd19ae0..9dcfbafb3 100644
--- a/test/test_cpu/test_autoround.py
+++ b/test/test_cpu/test_autoround.py
@@ -3,14 +3,13 @@ import shutil
import sys
import unittest
-from auto_round.eval.evaluation import simple_evaluate_user_model
-
sys.path.insert(0, "../..")
import torch
from _test_helpers import model_infer
from transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizer
from auto_round import AutoRound
+from auto_round.eval.evaluation import simple_evaluate_user_model
class LLMDataLoader:
@@ -606,6 +605,25 @@ class TestAutoRound(unittest.TestCase):
# inputs = tokenizer(text, return_tensors="pt").to(model.device)
# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))
+ def test_dequant_fp8_weight(self):
+ from auto_round.utils import dequant_block_fp8_weight
+
+ # test pad and unpad
+ weight = torch.randn(587, 7168)
+ weight_scale = torch.randn(5, 56)
+ block_size = [128, 128]
+ dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)
+ self.assertEqual(dequant_weight.shape.numel(), 4207616)
+
+ # test experts are stacked.
+ weight = torch.randn([32, 5760, 1440])
+ weight_scale = torch.randn([32, 5760, 90])
+ block_size = [1, 16]
+ dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)
+ self.assertEqual(len(dequant_weight.shape), 3)
+ self.assertEqual(dequant_weight.shape[0], 32)
+ self.assertEqual(dequant_weight.shape.numel(), 32 * 5760 * 1440)
+
if __name__ == "__main__":
unittest.main()