Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fp8 exporting bugfix (#874)

Weiweif56804a10461
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py
index 7f069cb60..bfc916419 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8.py
@@ -109,11 +109,9 @@ def pack_layer(layer_name, model, data_type, device=None):
torch_dtype = torch.float8_e5m2
info = torch.finfo(torch_dtype)
if zp is not None:
- q_weight = (
- weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp.to(packing_device)
- if isinstance(zp, torch.Tensor)
- else zp
- )
+ if isinstance(zp, torch.Tensor):
+ zp = zp.to(packing_device)
+ q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp
else:
q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)
q_weight = revert_tensor_by_pad(q_weight, orig_shape=orig_shape, pad_len=pad_len)
diff --git a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
index 4938f03f5..18931ffa2 100644
--- a/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
+++ b/auto_round/export/export_to_llmcompressor/export_to_static_fp.py
@@ -79,11 +79,9 @@ def pack_layer(layer_name: str, model: torch.nn.Module, data_type: str, device:
torch_dtype = torch.float8_e5m2
info = torch.finfo(torch_dtype)
if zp is not None:
- q_weight = (
- weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp.to(packing_device)
- if isinstance(zp, torch.Tensor)
- else zp
- )
+ if isinstance(zp, torch.Tensor):
+ zp = zp.to(packing_device)
+ q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1) + zp
else:
q_weight = weight.to(packing_device) / scale.to(packing_device).unsqueeze(-1)
q_weight = revert_tensor_by_pad(q_weight, orig_shape=orig_shape, pad_len=pad_len)