Skip to content

PTQ-Level Efficiency, QAT-Level Accuracy | 训练后量化的效率,量化感知训练的精度

fp8_block bug fix (#1693)

Wang, Mengni42101801bf11
diff --git a/auto_round/export/export_to_autoround/export_to_fp8.py b/auto_round/export/export_to_autoround/export_to_fp8.py
index fadebba5d..38af5b7b5 100644
--- a/auto_round/export/export_to_autoround/export_to_fp8.py
+++ b/auto_round/export/export_to_autoround/export_to_fp8.py
@@ -175,7 +175,7 @@ def pack_layer(layer_name, model, data_type, device=None, unsqueeze=False):
)
if (
unsqueeze
- and isinstance(linear_cls, FP8QLinear)
+ and isinstance(my_linear, FP8QLinear)
and len(my_linear.weight_scale.shape)
and my_linear.weight_scale.shape[0] != 1
):
diff --git a/test/test_cpu/export/test_llmc_format.py b/test/test_cpu/export/test_llmc_format.py
index 47db2b0f8..c6114c0df 100644
--- a/test/test_cpu/export/test_llmc_format.py
+++ b/test/test_cpu/export/test_llmc_format.py
@@ -58,12 +58,17 @@ class TestLLMC:
import json
+ from safetensors import safe_open
+
config = json.load(open(os.path.join(tmp_path, "config.json")))
assert "group_0" in config["quantization_config"]["config_groups"]
assert config["quantization_config"]["config_groups"]["group_0"]["input_activations"]["num_bits"] == 8
assert config["quantization_config"]["config_groups"]["group_0"]["weights"]["strategy"] == "channel"
assert config["quantization_config"]["quant_method"] == "compressed-tensors"
+ f = safe_open(os.path.join(tmp_path, "model.safetensors"), framework="pt")
+ assert len(f.get_tensor("model.decoder.layers.0.fc1.weight_scale").shape) == 2
+
def test_autoround_llmcompressor_fp8(self, tmp_path):
## quantize the model
model_name = opt_name_or_path