PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix bug of loading fp8 model (#747)
Heng Guo0029322760d0
diff --git a/auto_round/utils.py b/auto_round/utils.pyindex 2b8ef540e..4f47a3aa4 100644--- a/auto_round/utils.py+++ b/auto_round/utils.py@@ -24,6 +24,7 @@ import sysfrom collections import UserDictfrom enum import Enumfrom functools import lru_cache+from typing import Any, Tuple, Unionimport cpuinfoimport psutil@@ -2133,12 +2134,55 @@ def check_need_act_calibration(is_act_dynamic, act_data_type=None):return False-def dequant_block_fp8_weight(weight, weight_scale, block_size):+def pad_weight(weight: torch.Tensor, block_size: list) -> Tuple[torch.Tensor, int, int]:+ """Pads a matrix to make its dimensions multiples of block_size."""+ M, N = weight.shape[-2:]+ block_size_m, block_size_n = block_size+ pad_M = (block_size_m - M % block_size_m) % block_size_m+ pad_N = (block_size_n - N % block_size_n) % block_size_n++ if pad_M == 0 and pad_N == 0:+ return weight, M, N # No padding needed+ padded_weight = torch.nn.functional.pad(weight, (0, pad_N, 0, pad_M), mode="constant", value=0)+ return padded_weight, M, N # Return original dimensions for unpadding+++def unpad_weight(weight: torch.Tensor, original_M: int, original_N: int, keep_first_dim: bool = False) -> torch.Tensor:+ """Removes padding from the matrix to restore its original shape."""+ if (weight.shape[-2] == original_M) and (weight.shape[-1] == original_N):+ return weight+ if keep_first_dim:+ return weight[:, :original_M, :original_N]+ else:+ return weight[:original_M, :original_N]+++def pad_block_fp8_weight_naive(+ weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list+) -> Tuple[torch.Tensor, int, int]:++ assert len(block_size) == 2++ block_size_m, block_size_n = block_size+ weight_scale_m, weight_scale_n = weight_scale.shape[-2:]++ weight, orig_M, orig_N = pad_weight(weight, block_size)+ M, N = weight.shape[-2:]++ assert weight_scale_m == M // block_size_m+ assert weight_scale_n == N // block_size_n++ return weight, orig_M, orig_N+++def dequant_block_fp8_weight(weight: torch.Tensor, weight_scale: torch.Tensor, block_size: list) -> torch.Tensor:dtype = torch.bfloat16if weight_scale is None:return weightassert len(block_size) == 2+ weight, orig_M, orig_N = pad_block_fp8_weight_naive(weight, weight_scale, block_size)+weight_shape_len = len(weight.shape)block_size_m, block_size_n = block_size@@ -2150,15 +2194,19 @@ def dequant_block_fp8_weight(weight, weight_scale, block_size):weight = weight.view(weight_scale_m, block_size_m, weight_scale_n, block_size_n)dequant_weight = weight.to(dtype) * weight_scale.to(dtype)dequant_weight = dequant_weight.view(weight_scale_m * block_size_m, weight_scale_n * block_size_n)+ keep_first_dim = Falseelif weight_shape_len == 3:fd, weight_scale_m, weight_scale_n = weight_scale.shapeweight_scale = weight_scale.view(fd, weight_scale_m, 1, weight_scale_n, 1)weight = weight.view(fd, weight_scale_m, block_size_m, weight_scale_n, block_size_n)dequant_weight = weight.to(dtype) * weight_scale.to(dtype)dequant_weight = dequant_weight.view(fd, weight_scale_m * block_size_m, weight_scale_n * block_size_n)+ keep_first_dim = Trueelse:raise ValueError("Only support original weight shape is either 2 or 3")+ dequant_weight = unpad_weight(dequant_weight, orig_M, orig_N, keep_first_dim=keep_first_dim)+return dequant_weightdiff --git a/test/test_cpu/test_autoround.py b/test/test_cpu/test_autoround.pyindex afbd19ae0..9dcfbafb3 100644--- a/test/test_cpu/test_autoround.py+++ b/test/test_cpu/test_autoround.py@@ -3,14 +3,13 @@ import shutilimport sysimport unittest-from auto_round.eval.evaluation import simple_evaluate_user_model-sys.path.insert(0, "../..")import torchfrom _test_helpers import model_inferfrom transformers import AutoModelForCausalLM, AutoRoundConfig, AutoTokenizerfrom auto_round import AutoRound+from auto_round.eval.evaluation import simple_evaluate_user_modelclass LLMDataLoader:@@ -606,6 +605,25 @@ class TestAutoRound(unittest.TestCase):# inputs = tokenizer(text, return_tensors="pt").to(model.device)# print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0]))+ def test_dequant_fp8_weight(self):+ from auto_round.utils import dequant_block_fp8_weight++ # test pad and unpad+ weight = torch.randn(587, 7168)+ weight_scale = torch.randn(5, 56)+ block_size = [128, 128]+ dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)+ self.assertEqual(dequant_weight.shape.numel(), 4207616)++ # test experts are stacked.+ weight = torch.randn([32, 5760, 1440])+ weight_scale = torch.randn([32, 5760, 90])+ block_size = [1, 16]+ dequant_weight = dequant_block_fp8_weight(weight, weight_scale, block_size)+ self.assertEqual(len(dequant_weight.shape), 3)+ self.assertEqual(dequant_weight.shape[0], 32)+ self.assertEqual(dequant_weight.shape.numel(), 32 * 5760 * 1440)+if __name__ == "__main__":unittest.main()