PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度
fix gguf block-wise issue (#664)
Wenhua Cheng313f436ea935
diff --git a/auto_round/autoround.py b/auto_round/autoround.pyindex a0c675490..cd27447e0 100644--- a/auto_round/autoround.py+++ b/auto_round/autoround.py@@ -1172,7 +1172,7 @@ class AutoRound(object):for block_name in block_names:pbar.set_description(f"Quantizing {block_name}")block = get_module(self.model, block_name)-+ block = block.to(self.device)# Dispatch model if neededif self.device_map is not None:from accelerate import dispatch_modeldiff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.pyindex 2c2192efe..af115311d 100644--- a/auto_round/data_type/mxfp.py+++ b/auto_round/data_type/mxfp.py@@ -130,4 +130,4 @@ if __name__ == "__main__":data_neg = data * -1data2 = quant_element(data_neg, 2, 3, 6.0)- assert (torch.sum(torch.abs(data2 - gt * -1)) < 1e-6)+ assert (torch.sum(torch.abs(data2 - gt * -1)) < 1e-6)\ No newline at end of filediff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.pyindex 3a15f9b37..5265d55fc 100644--- a/auto_round/data_type/utils.py+++ b/auto_round/data_type/utils.py@@ -160,6 +160,7 @@ def floor_ste(x: torch.Tensor):return (x.floor() - x).detach() + x+def float8_e4m3fn_ste(x: torch.Tensor):"""Straight-Through Estimator (STE) for float8.