Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fix gguf block-wise issue (#664)

Wenhua Cheng313f436ea935
diff --git a/auto_round/autoround.py b/auto_round/autoround.py
index a0c675490..cd27447e0 100644
--- a/auto_round/autoround.py
+++ b/auto_round/autoround.py
@@ -1172,7 +1172,7 @@ class AutoRound(object):
for block_name in block_names:
pbar.set_description(f"Quantizing {block_name}")
block = get_module(self.model, block_name)
-
+ block = block.to(self.device)
# Dispatch model if needed
if self.device_map is not None:
from accelerate import dispatch_model
diff --git a/auto_round/data_type/mxfp.py b/auto_round/data_type/mxfp.py
index 2c2192efe..af115311d 100644
--- a/auto_round/data_type/mxfp.py
+++ b/auto_round/data_type/mxfp.py
@@ -130,4 +130,4 @@ if __name__ == "__main__":
data_neg = data * -1
data2 = quant_element(data_neg, 2, 3, 6.0)
- assert (torch.sum(torch.abs(data2 - gt * -1)) < 1e-6)
+ assert (torch.sum(torch.abs(data2 - gt * -1)) < 1e-6)
\ No newline at end of file
diff --git a/auto_round/data_type/utils.py b/auto_round/data_type/utils.py
index 3a15f9b37..5265d55fc 100644
--- a/auto_round/data_type/utils.py
+++ b/auto_round/data_type/utils.py
@@ -160,6 +160,7 @@ def floor_ste(x: torch.Tensor):
return (x.floor() - x).detach() + x
+
def float8_e4m3fn_ste(x: torch.Tensor):
"""Straight-Through Estimator (STE) for float8.